From f41cad9f3dd302b85c1c35afbd3abf22475d65df Mon Sep 17 00:00:00 2001 From: Luca Forstner Date: Wed, 19 Aug 2026 17:42:36 +0200 Subject: [PATCH 1/8] ref(v4)!: Remove flue pre-v1 support (#2380) ref [braintrustdata/braintrust-sdk-javascript#1841](https://github.com/braintrustdata/braintrust-sdk-javascript/issues/1841) --- .changeset/remove-flue-pre-v1.md | 5 + e2e/config/pr-comment-scenarios.json | 8 - .../flue-instrumentation/.flue/app.mjs | 82 +- .../.flue/workflows/instrumentation.mjs | 182 +- .../__cassettes__/flue-v0-8-0.cassette.json | 3727 ----------- .../flue-v0-8-latest.cassette.json | 5859 ----------------- .../flue-v0-8-0-auto-hook.span-tree.json | 1448 ---- .../flue-v0-8-0-auto-hook.span-tree.txt | 1335 ---- .../flue-v0-8-0-cli.span-tree.json | 1448 ---- .../flue-v0-8-0-cli.span-tree.txt | 1335 ---- .../flue-v0-8-0-explicit.span-tree.json | 1448 ---- .../flue-v0-8-0-explicit.span-tree.txt | 1335 ---- .../flue-v0-8-latest-auto-hook.span-tree.json | 1662 ----- .../flue-v0-8-latest-auto-hook.span-tree.txt | 1499 ----- .../flue-v0-8-latest-cli.span-tree.json | 1662 ----- .../flue-v0-8-latest-cli.span-tree.txt | 1499 ----- .../flue-v0-8-latest-explicit.span-tree.json | 1662 ----- .../flue-v0-8-latest-explicit.span-tree.txt | 1499 ----- ...flue-v1-0-0-beta-3-explicit.span-tree.json | 4 +- .../flue-v1-0-0-beta-3-explicit.span-tree.txt | 4 +- .../flue-v1-latest-explicit.span-tree.json | 4 +- .../flue-v1-latest-explicit.span-tree.txt | 4 +- .../flue-instrumentation/assertions.ts | 142 +- .../flue-instrumentation/package.json | 20 +- .../flue-instrumentation/pnpm-lock.yaml | 364 +- .../flue-instrumentation/scenario.cli.mjs | 4 - .../flue-instrumentation/scenario.impl.mjs | 115 +- .../flue-instrumentation/scenario.mjs | 8 - .../flue-instrumentation/scenario.test.ts | 119 +- .../flue-instrumentation/scenario.ts | 4 +- .../versions/v1/.flue/app.mjs | 112 - .../v1/.flue/workflows/instrumentation.mjs | 183 - js/src/auto-instrumentations/configs/all.ts | 5 - .../configs/flue.test.ts | 63 - js/src/auto-instrumentations/configs/flue.ts | 19 - js/src/exports.ts | 5 +- js/src/instrumentation/braintrust-plugin.ts | 12 - js/src/instrumentation/config.ts | 4 - js/src/instrumentation/index.ts | 5 +- .../instrumentation/plugins/flue-channels.ts | 14 - .../plugins/flue-plugin.test.ts | 558 +- js/src/instrumentation/plugins/flue-plugin.ts | 361 +- .../plugins/instrumentation-names.test.ts | 2 - js/src/vendor-sdk-types/flue.ts | 70 +- 44 files changed, 364 insertions(+), 29536 deletions(-) create mode 100644 .changeset/remove-flue-pre-v1.md delete mode 100644 e2e/scenarios/flue-instrumentation/__cassettes__/flue-v0-8-0.cassette.json delete mode 100644 e2e/scenarios/flue-instrumentation/__cassettes__/flue-v0-8-latest.cassette.json delete mode 100644 e2e/scenarios/flue-instrumentation/__snapshots__/flue-v0-8-0-auto-hook.span-tree.json delete mode 100644 e2e/scenarios/flue-instrumentation/__snapshots__/flue-v0-8-0-auto-hook.span-tree.txt delete mode 100644 e2e/scenarios/flue-instrumentation/__snapshots__/flue-v0-8-0-cli.span-tree.json delete mode 100644 e2e/scenarios/flue-instrumentation/__snapshots__/flue-v0-8-0-cli.span-tree.txt delete mode 100644 e2e/scenarios/flue-instrumentation/__snapshots__/flue-v0-8-0-explicit.span-tree.json delete mode 100644 e2e/scenarios/flue-instrumentation/__snapshots__/flue-v0-8-0-explicit.span-tree.txt delete mode 100644 e2e/scenarios/flue-instrumentation/__snapshots__/flue-v0-8-latest-auto-hook.span-tree.json delete mode 100644 e2e/scenarios/flue-instrumentation/__snapshots__/flue-v0-8-latest-auto-hook.span-tree.txt delete mode 100644 e2e/scenarios/flue-instrumentation/__snapshots__/flue-v0-8-latest-cli.span-tree.json delete mode 100644 e2e/scenarios/flue-instrumentation/__snapshots__/flue-v0-8-latest-cli.span-tree.txt delete mode 100644 e2e/scenarios/flue-instrumentation/__snapshots__/flue-v0-8-latest-explicit.span-tree.json delete mode 100644 e2e/scenarios/flue-instrumentation/__snapshots__/flue-v0-8-latest-explicit.span-tree.txt delete mode 100644 e2e/scenarios/flue-instrumentation/scenario.cli.mjs delete mode 100644 e2e/scenarios/flue-instrumentation/scenario.mjs delete mode 100644 e2e/scenarios/flue-instrumentation/versions/v1/.flue/app.mjs delete mode 100644 e2e/scenarios/flue-instrumentation/versions/v1/.flue/workflows/instrumentation.mjs delete mode 100644 js/src/auto-instrumentations/configs/flue.test.ts delete mode 100644 js/src/auto-instrumentations/configs/flue.ts delete mode 100644 js/src/instrumentation/plugins/flue-channels.ts diff --git a/.changeset/remove-flue-pre-v1.md b/.changeset/remove-flue-pre-v1.md new file mode 100644 index 000000000..a029265f8 --- /dev/null +++ b/.changeset/remove-flue-pre-v1.md @@ -0,0 +1,5 @@ +--- +"braintrust": major +--- + +ref(v4)!: Remove flue pre-v1 support diff --git a/e2e/config/pr-comment-scenarios.json b/e2e/config/pr-comment-scenarios.json index eb2b3f5f4..4399ab7e3 100644 --- a/e2e/config/pr-comment-scenarios.json +++ b/e2e/config/pr-comment-scenarios.json @@ -600,14 +600,6 @@ "label": "Flue Instrumentation", "metadataScenario": "flue-instrumentation", "variants": [ - { - "variantKey": "flue-v0-8-0", - "label": "v0.8 pinned" - }, - { - "variantKey": "flue-v0-8-latest", - "label": "v0.8 latest" - }, { "variantKey": "flue-v1-0-0-beta-3", "label": "v1 pinned" diff --git a/e2e/scenarios/flue-instrumentation/.flue/app.mjs b/e2e/scenarios/flue-instrumentation/.flue/app.mjs index 900402277..5d3ce1360 100644 --- a/e2e/scenarios/flue-instrumentation/.flue/app.mjs +++ b/e2e/scenarios/flue-instrumentation/.flue/app.mjs @@ -1,12 +1,12 @@ -import { mkdir, writeFile } from "node:fs/promises"; -import { dirname } from "node:path"; -import { flush, initLogger } from "braintrust"; +import { braintrustFlueInstrumentation, flush, initLogger } from "braintrust"; +import { Hono } from "hono"; const runtimePackageName = process.env.FLUE_RUNTIME_PACKAGE_NAME ?? "@flue/runtime"; -const { configureProvider, flue, observe } = await import( - `${runtimePackageName}/app` -); +const [{ instrument, registerProvider }, { flue }] = await Promise.all([ + import(runtimePackageName), + import(`${runtimePackageName}/routing`), +]); function projectName() { const configured = process.env.BRAINTRUST_E2E_PROJECT_NAME; @@ -19,84 +19,24 @@ function projectName() { initLogger({ projectName: projectName() }); -const exitProcess = process.exit.bind(process); -if (process.env.FLUE_E2E_FLUSH_FILE) { - let isExiting = false; - process.exit = (code) => { - if (isExiting) { - return exitProcess(code); - } - isExiting = true; - const keepAlive = setTimeout(() => {}, 30_000); - void flushBeforeExit() - .catch((error) => { - console.error(error); - }) - .finally(() => { - clearTimeout(keepAlive); - exitProcess(code); - }); - }; -} - -if (process.env.FLUE_E2E_EXPLICIT_OBSERVE === "1") { - const { braintrustFlueObserver } = await import("braintrust"); - observe(braintrustFlueObserver); -} +instrument(braintrustFlueInstrumentation()); const openAIBaseUrl = process.env.OPENAI_BASE_URL ?? process.env.BRAINTRUST_E2E_MODEL_BASE_URL; if (openAIBaseUrl) { - configureProvider("openai", { baseUrl: openAIBaseUrl }); + registerProvider("openai", { baseUrl: openAIBaseUrl }); } const anthropicBaseUrl = process.env.ANTHROPIC_BASE_URL; if (anthropicBaseUrl) { - configureProvider("anthropic", { + registerProvider("anthropic", { apiKey: process.env.ANTHROPIC_API_KEY ?? "test-key", baseUrl: anthropicBaseUrl, }); } -let didScheduleFlush = false; -function scheduleFinalFlush(exitAfterFlush = false) { - if (didScheduleFlush) { - return; - } - didScheduleFlush = true; - const keepAlive = setTimeout(() => {}, 30_000); - void flushBeforeExit() - .catch((error) => { - console.error(error); - }) - .finally(() => { - clearTimeout(keepAlive); - if (exitAfterFlush) { - exitProcess(0); - } - }); -} - -process.on("SIGTERM", () => { - scheduleFinalFlush(true); -}); - -process.on("beforeExit", () => { - scheduleFinalFlush(); -}); - -const app = flue(); - -async function flushBeforeExit() { - await new Promise((resolve) => setTimeout(resolve, 250)); - await flush(); - if (process.env.FLUE_E2E_FLUSH_FILE) { - await mkdir(dirname(process.env.FLUE_E2E_FLUSH_FILE), { - recursive: true, - }); - await writeFile(process.env.FLUE_E2E_FLUSH_FILE, "ok"); - } -} +const app = new Hono(); +app.route("/", flue()); export default { async fetch(request, env, ctx) { diff --git a/e2e/scenarios/flue-instrumentation/.flue/workflows/instrumentation.mjs b/e2e/scenarios/flue-instrumentation/.flue/workflows/instrumentation.mjs index dd15a5516..509be2b53 100644 --- a/e2e/scenarios/flue-instrumentation/.flue/workflows/instrumentation.mjs +++ b/e2e/scenarios/flue-instrumentation/.flue/workflows/instrumentation.mjs @@ -1,4 +1,5 @@ import { traced } from "braintrust"; +import * as v from "valibot"; import { FLUE_MODEL, FLUE_REASONING_MODEL, @@ -7,10 +8,11 @@ import { const runtimePackageName = process.env.FLUE_RUNTIME_PACKAGE_NAME ?? "@flue/runtime"; -const [{ createAgent, Type }, { local }] = await Promise.all([ - import(runtimePackageName), - import(`${runtimePackageName}/node`), -]); +const [{ defineAgent, defineTool, defineWorkflow }, { local }] = + await Promise.all([ + import(runtimePackageName), + import(`${runtimePackageName}/node`), + ]); function flueModel() { return process.env.FLUE_E2E_MODEL ?? FLUE_MODEL; @@ -34,7 +36,7 @@ function flueReasoningThinkingLevel() { return process.env.FLUE_E2E_REASONING_THINKING_LEVEL ?? "medium"; } -const flueE2EAgent = createAgent(() => ({ +const flueE2EAgent = defineAgent(() => ({ compaction: { keepRecentTokens: 1, reserveTokens: 64, @@ -51,10 +53,14 @@ const flueE2EAgent = createAgent(() => ({ thinkingLevel: "off", })); -const lookupTool = { +const lookupTool = defineTool({ description: "Return a deterministic lookup result with an id needed by web_search.", - execute: async (args) => { + input: v.object({ + query: v.string(), + }), + name: "lookup", + run: async ({ input }) => { await traced( async (span) => { span.log({ output: "lookup-active" }); @@ -69,109 +75,109 @@ const lookupTool = { }, ); - return JSON.stringify({ + return { id: "flue-session-2026", - query: args.query, + query: input.query, topic: "session instrumentation", - }); + }; }, - name: "lookup", - parameters: Type.Object({ - query: Type.String(), - }), -}; +}); -const webSearchTool = { +const webSearchTool = defineTool({ description: "Search a deterministic local web index. Requires the id returned by lookup.", - execute: async (args) => - JSON.stringify({ - lookupId: args.lookupId, - query: args.query, - results: [ - { - title: "Flue reasoning stream instrumentation", - url: "https://example.test/flue/reasoning-streams", - }, - ], - }), + input: v.object({ + lookupId: v.string(), + query: v.string(), + }), name: "web_search", - parameters: Type.Object({ - lookupId: Type.String(), - query: Type.String(), + run: async ({ input }) => ({ + lookupId: input.lookupId, + query: input.query, + results: [ + { + title: "Flue reasoning stream instrumentation", + url: "https://example.test/flue/reasoning-streams", + }, + ], }), -}; +}); -const summarizeSourceTool = { +const summarizeSourceTool = defineTool({ description: "Summarize the selected deterministic source after web_search returns a URL.", - execute: async (args) => - JSON.stringify({ - summary: - "Flue emits reasoning, tool execution, and LLM turn events separately.", - url: args.url, - }), + input: v.object({ + url: v.string(), + }), name: "summarize_source", - parameters: Type.Object({ - url: Type.String(), + run: async ({ input }) => ({ + summary: + "Flue emits reasoning, tool execution, and LLM turn events separately.", + url: input.url, }), -}; +}); export async function route(_ctx, next) { await next(); } -export async function run({ init, payload }) { - await traced( - async (span) => { - span.log({ output: "active" }); - }, - { - name: "flue.workflowCurrentProbe", - event: { - metadata: { - scenario: SCENARIO_NAME, +export default defineWorkflow({ + agent: flueE2EAgent, + input: v.object({ + scenario: v.optional(v.string()), + metadata: v.optional(v.record(v.string(), v.unknown())), + }), + async run({ harness, input }) { + await traced( + async (span) => { + span.log({ output: "active" }); + }, + { + name: "flue.workflowCurrentProbe", + event: { + metadata: { + scenario: SCENARIO_NAME, + }, }, }, - }, - ); + ); - const harness = await init(flueE2EAgent, { name: "default" }); - const session = await harness.session("main"); - const skillSession = await harness.session("skill"); - const taskSession = await harness.session("task"); - - await session.prompt( - [ - "Complete this instrumented research flow.", - "Call exactly one tool per turn and wait for each tool result before choosing the next tool.", - 'Step 1: call lookup with query "flue instrumentation".', - 'Step 2: use the lookup result id as lookupId and call web_search with query "Braintrust Flue reasoning stream instrumentation".', - "Step 3: use the first web_search result url and call summarize_source.", - "After summarize_source returns, reply with exactly PROMPT_DONE and no other text.", - ].join(" "), - { - model: fluePromptModel(), - thinkingLevel: fluePromptThinkingLevel(), - tools: [lookupTool, webSearchTool, summarizeSourceTool], - }, - ); + const session = await harness.session("main"); + const skillSession = await harness.session("skill"); + const taskSession = await harness.session("task"); + + await session.prompt( + [ + "Complete this instrumented research flow.", + "Call exactly one tool per turn and wait for each tool result before choosing the next tool.", + 'Step 1: call lookup with query "flue instrumentation".', + 'Step 2: use the lookup result id as lookupId and call web_search with query "Braintrust Flue reasoning stream instrumentation".', + "Step 3: use the first web_search result url and call summarize_source.", + "After summarize_source returns, reply with exactly PROMPT_DONE and no other text.", + ].join(" "), + { + model: fluePromptModel(), + thinkingLevel: fluePromptThinkingLevel(), + tools: [lookupTool, webSearchTool, summarizeSourceTool], + }, + ); - await skillSession.skill("e2e-flue-skill", { - args: { marker: "SKILL_DONE" }, - model: flueReasoningModel(), - thinkingLevel: "off", - }); + await skillSession.skill("e2e-flue-skill", { + args: { marker: "SKILL_DONE" }, + model: flueReasoningModel(), + thinkingLevel: "off", + }); - await taskSession.task("Reply with exactly TASK_DONE and no other text.", { - model: FLUE_MODEL, - thinkingLevel: "off", - }); + await taskSession.task("Reply with exactly TASK_DONE and no other text.", { + model: FLUE_MODEL, + thinkingLevel: "off", + }); - await session.compact(); + await session.compact(); - return { - scenario: payload?.scenario ?? SCENARIO_NAME, - status: "done", - }; -} + return { + scenario: input.scenario ?? SCENARIO_NAME, + status: "done", + }; + }, +}); diff --git a/e2e/scenarios/flue-instrumentation/__cassettes__/flue-v0-8-0.cassette.json b/e2e/scenarios/flue-instrumentation/__cassettes__/flue-v0-8-0.cassette.json deleted file mode 100644 index 996df5098..000000000 --- a/e2e/scenarios/flue-instrumentation/__cassettes__/flue-v0-8-0.cassette.json +++ /dev/null @@ -1,3727 +0,0 @@ -{ - "entries": [ - { - "callIndex": 0, - "id": "fcd9d51c145fb90b", - "matchKey": "POST api.anthropic.com/v1/messages", - "recordedAt": "2026-05-28T13:40:49.578Z", - "request": { - "body": { - "kind": "json", - "value": { - "max_tokens": 64000, - "messages": [ - { - "content": [ - { - "cache_control": { - "type": "ephemeral" - }, - "text": "Complete this instrumented research flow. Call exactly one tool per turn and wait for each tool result before choosing the next tool. Step 1: call lookup with query \"flue instrumentation\". Step 2: use the lookup result id as lookupId and call web_search with query \"Braintrust Flue reasoning stream instrumentation\". Step 3: use the first web_search result url and call summarize_source. After summarize_source returns, reply with exactly PROMPT_DONE and no other text.", - "type": "text" - } - ], - "role": "user" - } - ], - "model": "claude-sonnet-4-5-20250929", - "stream": true, - "system": [ - { - "cache_control": { - "type": "ephemeral" - }, - "text": "You are running in headless mode with no human operator. Work autonomously — never ask questions, never wait for user input. Make your best judgment and proceed independently.\n\nYou are a deterministic Flue instrumentation test agent. Follow user instructions exactly. When asked for a marker, output only that marker and no extra text. When running a local skill file, read it yourself and do not delegate it to a task.\n\n## Available Skills\n\nThe following skills provide specialized instructions for specific tasks. When a task matches a skill description, activate that skill before proceeding so its full instructions are loaded. Skill instructions and supporting resources stay lazy until activation or explicit file reads.\n\n- **e2e-flue-skill** — Deterministic skill used by the Braintrust Flue instrumentation e2e test.\n\nDate: Thu, May 28, 2026\nWorking directory: /Users/lucaforstner/conductor/workspaces/braintrust-sdk-javascript/shanghai-v1/e2e/.bt-tmp/run-u9uXnv/scenarios/flue-instrumentation\n\nDirectory structure:\n.agents\n__cassettes__\n__snapshots__\nassertions.ts\ncassette-filter.mjs\nconstants.mjs\nnode_modules\npackage.json\npnpm-lock.yaml\nscenario.impl.mjs\nscenario.mjs\nscenario.test.ts\nscenario.ts", - "type": "text" - } - ], - "thinking": { - "budget_tokens": 8192, - "display": "summarized", - "type": "enabled" - }, - "tools": [ - { - "description": "Read a file or list a directory. For files, output is truncated to 2000 lines or 50KB — use offset/limit for large files. For directories, returns the list of entries.", - "eager_input_streaming": true, - "input_schema": { - "properties": { - "limit": { - "description": "Maximum number of lines to read", - "type": "number" - }, - "offset": { - "description": "Line number to start from (1-indexed)", - "type": "number" - }, - "path": { - "description": "Path to the file to read", - "type": "string" - } - }, - "required": ["path"], - "type": "object" - }, - "name": "read" - }, - { - "description": "Write content to a file. Creates the file and parent directories if they do not exist.", - "eager_input_streaming": true, - "input_schema": { - "properties": { - "content": { - "description": "Content to write to the file", - "type": "string" - }, - "path": { - "description": "Path to the file to write", - "type": "string" - } - }, - "required": ["path", "content"], - "type": "object" - }, - "name": "write" - }, - { - "description": "Edit a file using exact text replacement. The oldText must match a unique region of the file. Use replaceAll to replace all occurrences.", - "eager_input_streaming": true, - "input_schema": { - "properties": { - "newText": { - "description": "Replacement text", - "type": "string" - }, - "oldText": { - "description": "Exact text to find (must be unique)", - "type": "string" - }, - "path": { - "description": "Path to the file to edit", - "type": "string" - }, - "replaceAll": { - "description": "Replace all occurrences", - "type": "boolean" - } - }, - "required": ["path", "oldText", "newText"], - "type": "object" - }, - "name": "edit" - }, - { - "description": "Execute a bash command. Returns stdout and stderr. Output is truncated to the last 2000 lines or 50KB.", - "eager_input_streaming": true, - "input_schema": { - "properties": { - "command": { - "description": "Bash command to execute", - "type": "string" - }, - "timeout": { - "description": "Timeout in seconds", - "type": "number" - } - }, - "required": ["command"], - "type": "object" - }, - "name": "bash" - }, - { - "description": "Search file contents for a regex pattern. Returns matching lines with file paths and line numbers.", - "eager_input_streaming": true, - "input_schema": { - "properties": { - "include": { - "description": "Glob filter, e.g. \"*.ts\"", - "type": "string" - }, - "path": { - "description": "Directory or file to search (default: .)", - "type": "string" - }, - "pattern": { - "description": "Search pattern (regex)", - "type": "string" - } - }, - "required": ["pattern"], - "type": "object" - }, - "name": "grep" - }, - { - "description": "Find files by filename pattern using shell find -name semantics. Returns matching file paths.", - "eager_input_streaming": true, - "input_schema": { - "properties": { - "path": { - "description": "Directory to search in (default: .)", - "type": "string" - }, - "pattern": { - "description": "Filename pattern, e.g. \"*.ts\"", - "type": "string" - } - }, - "required": ["pattern"], - "type": "object" - }, - "name": "glob" - }, - { - "description": "Delegate a focused task to a detached child agent with its own context. Use this for independent research, file exploration, or parallel work. The task returns only its final answer to this conversation. No subagents are currently defined.", - "eager_input_streaming": true, - "input_schema": { - "properties": { - "agent": { - "description": "Declared subagent to use for the child agent", - "type": "string" - }, - "cwd": { - "description": "Working directory for the child agent. AGENTS.md and skills are discovered from here.", - "type": "string" - }, - "description": { - "description": "Short human-readable label for the delegated work", - "type": "string" - }, - "prompt": { - "description": "Focused instructions for the child agent", - "type": "string" - } - }, - "required": ["prompt"], - "type": "object" - }, - "name": "task" - }, - { - "description": "Return a deterministic lookup result with an id needed by web_search.", - "eager_input_streaming": true, - "input_schema": { - "properties": { - "query": { - "type": "string" - } - }, - "required": ["query"], - "type": "object" - }, - "name": "lookup" - }, - { - "description": "Search a deterministic local web index. Requires the id returned by lookup.", - "eager_input_streaming": true, - "input_schema": { - "properties": { - "lookupId": { - "type": "string" - }, - "query": { - "type": "string" - } - }, - "required": ["lookupId", "query"], - "type": "object" - }, - "name": "web_search" - }, - { - "cache_control": { - "type": "ephemeral" - }, - "description": "Summarize the selected deterministic source after web_search returns a URL.", - "eager_input_streaming": true, - "input_schema": { - "properties": { - "url": { - "type": "string" - } - }, - "required": ["url"], - "type": "object" - }, - "name": "summarize_source" - } - ] - } - }, - "headers": {}, - "method": "POST", - "url": "https://api.anthropic.com/v1/messages" - }, - "response": { - "body": { - "chunks": [ - "event: message_start\ndata: {\"type\":\"message_start\",\"message\":{\"model\":\"claude-sonnet-4-5-20250929\",\"id\":\"msg_01A7K7rFFvpGEkYsKHme6rCx\",\"type\":\"message\",\"role\":\"assistant\",\"content\":[],\"stop_reason\":null,\"stop_sequence\":null,\"stop_details\":null,\"usage\":{\"input_tokens\":10,\"cache_creation_input_tokens\":2222,\"cache_read_input_tokens\":0,\"cache_creation\":{\"ephemeral_5m_input_tokens\":2222,\"ephemeral_1h_input_tokens\":0},\"output_tokens\":8,\"service_tier\":\"standard\",\"inference_geo\":\"not_available\"}}}", - "event: content_block_start\ndata: {\"type\":\"content_block_start\",\"index\":0,\"content_block\":{\"type\":\"thinking\",\"thinking\":\"\",\"signature\":\"\"}}", - "event: ping\ndata: {\"type\":\"ping\"}", - "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"thinking_delta\",\"thinking\":\"The user wants me to complete a research\"}}", - "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"thinking_delta\",\"thinking\":\" flow with exactly three steps:\\n1. Call lookup with query \\\"flue instrumentation\\\"\\n2. Call web_search with the\"}}", - "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"thinking_delta\",\"thinking\":\" lookupId from step 1 and query \\\"Braintrust Flue reasoning stream instrumentation\\\"\\n3. Call summarize_source with the first URL from step 2\"}}", - "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"thinking_delta\",\"thinking\":\"\\n4. Reply with exactly \\\"PROMPT_DONE\\\"\\n\\nThe user emphasizes I should call exactly one tool per turn and wait for each result\"}}", - "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"thinking_delta\",\"thinking\":\". Let me start with step 1.\"}}", - "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"signature_delta\",\"signature\":\"EoQFCm4IDhgCKkD+AoKMhpXNPUJ5eANwM28uUCJPCE+AXh0ACw7aXniX+YN04zdJl3hlHMbtAuKhhczLiEeOJtu4KtVpea5u5NFdMhpjbGF1ZGUtc29ubmV0LTQtNS0yMDI1MDkyOTgAQgh0aGlua2luZxIMIOTWCNEtsiI9vF4SGgwPXC861uZx8lpOziEiMLsCukGIluD0U44degzYnHFc+BwXNIEB+yatSHlZE+a5smr9crZcft3OLmsIdXuaFSrDAyQxWd9oqfJdnvza9eWIaOCeNkVCjrTc/wotBwh5Xp8m9k/WKt7P4zzUJBuLQ1f6ZnsdWkL8OiL/2E7Up55z2+UuVWsg1Y90/x27YRFdQjv3RnztYp4bZ5oIHAlTJoCzWiBYEh/Z8c2O+wy6MHo1cp0T57OroB/4HAqRZxrjTOXpKYLmYKXAY5tU8DTd3A0/FfEhg3oPSbOhX+112szFCpm3BRNJYooUUhwSu1hO4H/6r7d/Yp9XIM4tBfzuDx83CC/YetdOFgnYkdliw0xaUGaj4o37rWIRctWHAMdlUc4NDA0Ljc1uirHDXkN8EJwWSoIjPSBTA/p/wgLrvRV5Yy9Z0dMoWOpgYF6t1H/tNAyY8ml1CA7IWyd3jnfTZ1/G3Y0xFDwJDFlb4/QKc67w36SSCa+X9mMoTxD/DYN249lsII+XX3lQ7F1zJ07THOtCyRirCaRhmE/kGwl5tiB1QHrWa9bNHxQxf9i92435JGKN053Fe14AR70MtBSzIU2/zbgvhB1vF5lZt2i46hNxfo4lXR495gEwdwC6+8HJnlEJAJkdeHfZKj+RiwXoiYKPKsSbHi83X8Hh32IifuglNxdlxVQYAQ==\"}}", - "event: content_block_stop\ndata: {\"type\":\"content_block_stop\",\"index\":0}", - "event: content_block_start\ndata: {\"type\":\"content_block_start\",\"index\":1,\"content_block\":{\"type\":\"text\",\"text\":\"\"}}", - "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":1,\"delta\":{\"type\":\"text_delta\",\"text\":\"I'll complete this instrumented research flow step\"}}", - "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":1,\"delta\":{\"type\":\"text_delta\",\"text\":\" by step. Starting with Step 1:\"}}", - "event: content_block_stop\ndata: {\"type\":\"content_block_stop\",\"index\":1}", - "event: content_block_start\ndata: {\"type\":\"content_block_start\",\"index\":2,\"content_block\":{\"type\":\"tool_use\",\"id\":\"toolu_01MbotvqKgQV4ecxQ566vyuC\",\"name\":\"lookup\",\"input\":{},\"caller\":{\"type\":\"direct\"}}}", - "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":2,\"delta\":{\"type\":\"input_json_delta\",\"partial_json\":\"\"}}", - "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":2,\"delta\":{\"type\":\"input_json_delta\",\"partial_json\":\"{\\\"query\\\": \\\"flue instrumentation\"}}", - "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":2,\"delta\":{\"type\":\"input_json_delta\",\"partial_json\":\"\\\"}\"}}", - "event: content_block_stop\ndata: {\"type\":\"content_block_stop\",\"index\":2}", - "event: message_delta\ndata: {\"type\":\"message_delta\",\"delta\":{\"stop_reason\":\"tool_use\",\"stop_sequence\":null,\"stop_details\":null},\"usage\":{\"input_tokens\":10,\"cache_creation_input_tokens\":2222,\"cache_read_input_tokens\":0,\"output_tokens\":196,\"output_tokens_details\":{\"thinking_tokens\":120}}}", - "event: message_stop\ndata: {\"type\":\"message_stop\"}" - ], - "kind": "sse" - }, - "headers": { - "anthropic-organization-id": "27796668-7351-40ac-acc4-024aee8995a5", - "anthropic-ratelimit-input-tokens-limit": "3000000", - "anthropic-ratelimit-input-tokens-remaining": "2999000", - "anthropic-ratelimit-input-tokens-reset": "2026-05-28T13:40:45Z", - "anthropic-ratelimit-output-tokens-limit": "600000", - "anthropic-ratelimit-output-tokens-remaining": "600000", - "anthropic-ratelimit-output-tokens-reset": "2026-05-28T13:40:45Z", - "anthropic-ratelimit-requests-limit": "20000", - "anthropic-ratelimit-requests-remaining": "19999", - "anthropic-ratelimit-requests-reset": "2026-05-28T13:40:45Z", - "anthropic-ratelimit-tokens-limit": "3600000", - "anthropic-ratelimit-tokens-remaining": "3599000", - "anthropic-ratelimit-tokens-reset": "2026-05-28T13:40:45Z", - "cache-control": "no-cache", - "cf-cache-status": "DYNAMIC", - "cf-ray": "a02dac490ea0c301-VIE", - "connection": "keep-alive", - "content-encoding": "gzip", - "content-security-policy": "default-src 'none'; frame-ancestors 'none'", - "content-type": "text/event-stream; charset=utf-8", - "date": "Thu, 28 May 2026 13:40:47 GMT", - "request-id": "req_011CbV4Yrzdciu7aM7XyPJvu", - "server": "cloudflare", - "set-cookie": "[REDACTED]", - "strict-transport-security": "max-age=31536000; includeSubDomains; preload", - "traceresponse": "00-def621f99a586b9f40e35c15380d1f90-62267612b2e498ea-01", - "transfer-encoding": "chunked", - "vary": "Accept-Encoding", - "x-envoy-upstream-service-time": "1355", - "x-robots-tag": "none" - }, - "status": 200, - "statusText": "OK" - } - }, - { - "callIndex": 1, - "id": "ff22a182ea5a9c06", - "matchKey": "POST api.anthropic.com/v1/messages", - "recordedAt": "2026-05-28T13:40:53.810Z", - "request": { - "body": { - "kind": "json", - "value": { - "max_tokens": 64000, - "messages": [ - { - "content": [ - { - "text": "Complete this instrumented research flow. Call exactly one tool per turn and wait for each tool result before choosing the next tool. Step 1: call lookup with query \"flue instrumentation\". Step 2: use the lookup result id as lookupId and call web_search with query \"Braintrust Flue reasoning stream instrumentation\". Step 3: use the first web_search result url and call summarize_source. After summarize_source returns, reply with exactly PROMPT_DONE and no other text.", - "type": "text" - } - ], - "role": "user" - }, - { - "content": [ - { - "signature": "EoQFCm4IDhgCKkD+AoKMhpXNPUJ5eANwM28uUCJPCE+AXh0ACw7aXniX+YN04zdJl3hlHMbtAuKhhczLiEeOJtu4KtVpea5u5NFdMhpjbGF1ZGUtc29ubmV0LTQtNS0yMDI1MDkyOTgAQgh0aGlua2luZxIMIOTWCNEtsiI9vF4SGgwPXC861uZx8lpOziEiMLsCukGIluD0U44degzYnHFc+BwXNIEB+yatSHlZE+a5smr9crZcft3OLmsIdXuaFSrDAyQxWd9oqfJdnvza9eWIaOCeNkVCjrTc/wotBwh5Xp8m9k/WKt7P4zzUJBuLQ1f6ZnsdWkL8OiL/2E7Up55z2+UuVWsg1Y90/x27YRFdQjv3RnztYp4bZ5oIHAlTJoCzWiBYEh/Z8c2O+wy6MHo1cp0T57OroB/4HAqRZxrjTOXpKYLmYKXAY5tU8DTd3A0/FfEhg3oPSbOhX+112szFCpm3BRNJYooUUhwSu1hO4H/6r7d/Yp9XIM4tBfzuDx83CC/YetdOFgnYkdliw0xaUGaj4o37rWIRctWHAMdlUc4NDA0Ljc1uirHDXkN8EJwWSoIjPSBTA/p/wgLrvRV5Yy9Z0dMoWOpgYF6t1H/tNAyY8ml1CA7IWyd3jnfTZ1/G3Y0xFDwJDFlb4/QKc67w36SSCa+X9mMoTxD/DYN249lsII+XX3lQ7F1zJ07THOtCyRirCaRhmE/kGwl5tiB1QHrWa9bNHxQxf9i92435JGKN053Fe14AR70MtBSzIU2/zbgvhB1vF5lZt2i46hNxfo4lXR495gEwdwC6+8HJnlEJAJkdeHfZKj+RiwXoiYKPKsSbHi83X8Hh32IifuglNxdlxVQYAQ==", - "thinking": "The user wants me to complete a research flow with exactly three steps:\n1. Call lookup with query \"flue instrumentation\"\n2. Call web_search with the lookupId from step 1 and query \"Braintrust Flue reasoning stream instrumentation\"\n3. Call summarize_source with the first URL from step 2\n4. Reply with exactly \"PROMPT_DONE\"\n\nThe user emphasizes I should call exactly one tool per turn and wait for each result. Let me start with step 1.", - "type": "thinking" - }, - { - "text": "I'll complete this instrumented research flow step by step. Starting with Step 1:", - "type": "text" - }, - { - "id": "toolu_01MbotvqKgQV4ecxQ566vyuC", - "input": { - "query": "flue instrumentation" - }, - "name": "lookup", - "type": "tool_use" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "cache_control": { - "type": "ephemeral" - }, - "content": "{\"id\":\"flue-session-2026\",\"query\":\"flue instrumentation\",\"topic\":\"session instrumentation\"}", - "is_error": false, - "tool_use_id": "toolu_01MbotvqKgQV4ecxQ566vyuC", - "type": "tool_result" - } - ], - "role": "user" - } - ], - "model": "claude-sonnet-4-5-20250929", - "stream": true, - "system": [ - { - "cache_control": { - "type": "ephemeral" - }, - "text": "You are running in headless mode with no human operator. Work autonomously — never ask questions, never wait for user input. Make your best judgment and proceed independently.\n\nYou are a deterministic Flue instrumentation test agent. Follow user instructions exactly. When asked for a marker, output only that marker and no extra text. When running a local skill file, read it yourself and do not delegate it to a task.\n\n## Available Skills\n\nThe following skills provide specialized instructions for specific tasks. When a task matches a skill description, activate that skill before proceeding so its full instructions are loaded. Skill instructions and supporting resources stay lazy until activation or explicit file reads.\n\n- **e2e-flue-skill** — Deterministic skill used by the Braintrust Flue instrumentation e2e test.\n\nDate: Thu, May 28, 2026\nWorking directory: /Users/lucaforstner/conductor/workspaces/braintrust-sdk-javascript/shanghai-v1/e2e/.bt-tmp/run-u9uXnv/scenarios/flue-instrumentation\n\nDirectory structure:\n.agents\n__cassettes__\n__snapshots__\nassertions.ts\ncassette-filter.mjs\nconstants.mjs\nnode_modules\npackage.json\npnpm-lock.yaml\nscenario.impl.mjs\nscenario.mjs\nscenario.test.ts\nscenario.ts", - "type": "text" - } - ], - "thinking": { - "budget_tokens": 8192, - "display": "summarized", - "type": "enabled" - }, - "tools": [ - { - "description": "Read a file or list a directory. For files, output is truncated to 2000 lines or 50KB — use offset/limit for large files. For directories, returns the list of entries.", - "eager_input_streaming": true, - "input_schema": { - "properties": { - "limit": { - "description": "Maximum number of lines to read", - "type": "number" - }, - "offset": { - "description": "Line number to start from (1-indexed)", - "type": "number" - }, - "path": { - "description": "Path to the file to read", - "type": "string" - } - }, - "required": ["path"], - "type": "object" - }, - "name": "read" - }, - { - "description": "Write content to a file. Creates the file and parent directories if they do not exist.", - "eager_input_streaming": true, - "input_schema": { - "properties": { - "content": { - "description": "Content to write to the file", - "type": "string" - }, - "path": { - "description": "Path to the file to write", - "type": "string" - } - }, - "required": ["path", "content"], - "type": "object" - }, - "name": "write" - }, - { - "description": "Edit a file using exact text replacement. The oldText must match a unique region of the file. Use replaceAll to replace all occurrences.", - "eager_input_streaming": true, - "input_schema": { - "properties": { - "newText": { - "description": "Replacement text", - "type": "string" - }, - "oldText": { - "description": "Exact text to find (must be unique)", - "type": "string" - }, - "path": { - "description": "Path to the file to edit", - "type": "string" - }, - "replaceAll": { - "description": "Replace all occurrences", - "type": "boolean" - } - }, - "required": ["path", "oldText", "newText"], - "type": "object" - }, - "name": "edit" - }, - { - "description": "Execute a bash command. Returns stdout and stderr. Output is truncated to the last 2000 lines or 50KB.", - "eager_input_streaming": true, - "input_schema": { - "properties": { - "command": { - "description": "Bash command to execute", - "type": "string" - }, - "timeout": { - "description": "Timeout in seconds", - "type": "number" - } - }, - "required": ["command"], - "type": "object" - }, - "name": "bash" - }, - { - "description": "Search file contents for a regex pattern. Returns matching lines with file paths and line numbers.", - "eager_input_streaming": true, - "input_schema": { - "properties": { - "include": { - "description": "Glob filter, e.g. \"*.ts\"", - "type": "string" - }, - "path": { - "description": "Directory or file to search (default: .)", - "type": "string" - }, - "pattern": { - "description": "Search pattern (regex)", - "type": "string" - } - }, - "required": ["pattern"], - "type": "object" - }, - "name": "grep" - }, - { - "description": "Find files by filename pattern using shell find -name semantics. Returns matching file paths.", - "eager_input_streaming": true, - "input_schema": { - "properties": { - "path": { - "description": "Directory to search in (default: .)", - "type": "string" - }, - "pattern": { - "description": "Filename pattern, e.g. \"*.ts\"", - "type": "string" - } - }, - "required": ["pattern"], - "type": "object" - }, - "name": "glob" - }, - { - "description": "Delegate a focused task to a detached child agent with its own context. Use this for independent research, file exploration, or parallel work. The task returns only its final answer to this conversation. No subagents are currently defined.", - "eager_input_streaming": true, - "input_schema": { - "properties": { - "agent": { - "description": "Declared subagent to use for the child agent", - "type": "string" - }, - "cwd": { - "description": "Working directory for the child agent. AGENTS.md and skills are discovered from here.", - "type": "string" - }, - "description": { - "description": "Short human-readable label for the delegated work", - "type": "string" - }, - "prompt": { - "description": "Focused instructions for the child agent", - "type": "string" - } - }, - "required": ["prompt"], - "type": "object" - }, - "name": "task" - }, - { - "description": "Return a deterministic lookup result with an id needed by web_search.", - "eager_input_streaming": true, - "input_schema": { - "properties": { - "query": { - "type": "string" - } - }, - "required": ["query"], - "type": "object" - }, - "name": "lookup" - }, - { - "description": "Search a deterministic local web index. Requires the id returned by lookup.", - "eager_input_streaming": true, - "input_schema": { - "properties": { - "lookupId": { - "type": "string" - }, - "query": { - "type": "string" - } - }, - "required": ["lookupId", "query"], - "type": "object" - }, - "name": "web_search" - }, - { - "cache_control": { - "type": "ephemeral" - }, - "description": "Summarize the selected deterministic source after web_search returns a URL.", - "eager_input_streaming": true, - "input_schema": { - "properties": { - "url": { - "type": "string" - } - }, - "required": ["url"], - "type": "object" - }, - "name": "summarize_source" - } - ] - } - }, - "headers": {}, - "method": "POST", - "url": "https://api.anthropic.com/v1/messages" - }, - "response": { - "body": { - "chunks": [ - "event: message_start\ndata: {\"type\":\"message_start\",\"message\":{\"model\":\"claude-sonnet-4-5-20250929\",\"id\":\"msg_01HZbhxuwLjyFGr1czSAWHTh\",\"type\":\"message\",\"role\":\"assistant\",\"content\":[],\"stop_reason\":null,\"stop_sequence\":null,\"stop_details\":null,\"usage\":{\"input_tokens\":13,\"cache_creation_input_tokens\":236,\"cache_read_input_tokens\":2222,\"cache_creation\":{\"ephemeral_5m_input_tokens\":236,\"ephemeral_1h_input_tokens\":0},\"output_tokens\":1,\"service_tier\":\"standard\",\"inference_geo\":\"not_available\"}}}", - "event: content_block_start\ndata: {\"type\":\"content_block_start\",\"index\":0,\"content_block\":{\"type\":\"thinking\",\"thinking\":\"\",\"signature\":\"\"}}", - "event: ping\ndata: {\"type\":\"ping\"}", - "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"thinking_delta\",\"thinking\":\"Good\"}}", - "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"thinking_delta\",\"thinking\":\", I got the lookup result with id \\\"flue-session-2026\\\". Now I need to proceed to step 2:\"}}", - "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"thinking_delta\",\"thinking\":\" call web_search with this lookupId and the query \\\"Braintrust Flue reasoning stream instrumentation\\\".\"}}", - "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"signature_delta\",\"signature\":\"EpEDCm4IDhgCKkAMahve1kB1j4njp1ZSzO8eCK4bb5Ex7MB6tjdoOXfuUDRfkBqvIGdwVly8GiA8GGYC7Y/RVfREXF6A6OXHTwFyMhpjbGF1ZGUtc29ubmV0LTQtNS0yMDI1MDkyOTgAQgh0aGlua2luZxIMLw+fsa4yo7a7ZSoIGgyaTdvH020n2dQj9bUiMB4OokY29w+uZb4rqDnKDbLXUqUj6U0f1QCGQF2ZBYDDfDJyM5Jv1xY3rQiUUjkhBirQAS4XivjvMnWI5zHBmpI/u/pq4aGdrdb3vUsJ/ne6R1d80JA/INGF8vLkQcZ5xxEK0jU4yRE+ldc2uzavmtoxIKAFD6bqKks1Qv6u+TIZtuW7tUB/7jFxDw93KmJNruMIa8c1J6pPUyBy3xE7QwgmU0kDfIfdnljxLoEBiMwtnxjCPj9lT7HHz4FkrhbnQTV4MvAQohCLjaSbOTd6LdLxwtJPm9mc26ID4eYN1gCCzyAxGXhqLHlzc23PafvQjsZliWnLhaiNMnG0YEo78XkuJQsYAQ==\"}}", - "event: content_block_stop\ndata: {\"type\":\"content_block_stop\",\"index\":0}", - "event: content_block_start\ndata: {\"type\":\"content_block_start\",\"index\":1,\"content_block\":{\"type\":\"text\",\"text\":\"\"}}", - "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":1,\"delta\":{\"type\":\"text_delta\",\"text\":\"Now proceeding to Step 2:\"}}", - "event: content_block_stop\ndata: {\"type\":\"content_block_stop\",\"index\":1}", - "event: content_block_start\ndata: {\"type\":\"content_block_start\",\"index\":2,\"content_block\":{\"type\":\"tool_use\",\"id\":\"toolu_018rc1ig7vXeron3rLfpUtRT\",\"name\":\"web_search\",\"input\":{},\"caller\":{\"type\":\"direct\"}}}", - "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":2,\"delta\":{\"type\":\"input_json_delta\",\"partial_json\":\"\"}}", - "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":2,\"delta\":{\"type\":\"input_json_delta\",\"partial_json\":\"{\\\"lookupId\\\": \\\"flue-session-2026\"}}", - "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":2,\"delta\":{\"type\":\"input_json_delta\",\"partial_json\":\"\\\", \\\"query\\\": \\\"Braintrust Flue reasoning stream instru\"}}", - "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":2,\"delta\":{\"type\":\"input_json_delta\",\"partial_json\":\"mentation\"}}", - "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":2,\"delta\":{\"type\":\"input_json_delta\",\"partial_json\":\"\\\"}\"}}", - "event: content_block_stop\ndata: {\"type\":\"content_block_stop\",\"index\":2}", - "event: message_delta\ndata: {\"type\":\"message_delta\",\"delta\":{\"stop_reason\":\"tool_use\",\"stop_sequence\":null,\"stop_details\":null},\"usage\":{\"input_tokens\":13,\"cache_creation_input_tokens\":236,\"cache_read_input_tokens\":2222,\"output_tokens\":158,\"output_tokens_details\":{\"thinking_tokens\":59}}}", - "event: message_stop\ndata: {\"type\":\"message_stop\"}" - ], - "kind": "sse" - }, - "headers": { - "anthropic-organization-id": "27796668-7351-40ac-acc4-024aee8995a5", - "anthropic-ratelimit-input-tokens-limit": "3000000", - "anthropic-ratelimit-input-tokens-remaining": "2999000", - "anthropic-ratelimit-input-tokens-reset": "2026-05-28T13:40:49Z", - "anthropic-ratelimit-output-tokens-limit": "600000", - "anthropic-ratelimit-output-tokens-remaining": "600000", - "anthropic-ratelimit-output-tokens-reset": "2026-05-28T13:40:49Z", - "anthropic-ratelimit-requests-limit": "20000", - "anthropic-ratelimit-requests-remaining": "19999", - "anthropic-ratelimit-requests-reset": "2026-05-28T13:40:49Z", - "anthropic-ratelimit-tokens-limit": "3600000", - "anthropic-ratelimit-tokens-remaining": "3599000", - "anthropic-ratelimit-tokens-reset": "2026-05-28T13:40:49Z", - "cache-control": "no-cache", - "cf-cache-status": "DYNAMIC", - "cf-ray": "a02dac631dbac301-VIE", - "connection": "keep-alive", - "content-encoding": "gzip", - "content-security-policy": "default-src 'none'; frame-ancestors 'none'", - "content-type": "text/event-stream; charset=utf-8", - "date": "Thu, 28 May 2026 13:40:52 GMT", - "request-id": "req_011CbV4ZAqRr9QtjYnVyY69n", - "server": "cloudflare", - "set-cookie": "[REDACTED]", - "strict-transport-security": "max-age=31536000; includeSubDomains; preload", - "traceresponse": "00-68d6456f3077bdeb0822f73c39a51d2d-5be926a484ba813d-01", - "transfer-encoding": "chunked", - "vary": "Accept-Encoding", - "x-envoy-upstream-service-time": "2225", - "x-robots-tag": "none" - }, - "status": 200, - "statusText": "OK" - } - }, - { - "callIndex": 2, - "id": "f1d50a1e6c760d6c", - "matchKey": "POST api.anthropic.com/v1/messages", - "recordedAt": "2026-05-28T13:40:58.541Z", - "request": { - "body": { - "kind": "json", - "value": { - "max_tokens": 64000, - "messages": [ - { - "content": [ - { - "text": "Complete this instrumented research flow. Call exactly one tool per turn and wait for each tool result before choosing the next tool. Step 1: call lookup with query \"flue instrumentation\". Step 2: use the lookup result id as lookupId and call web_search with query \"Braintrust Flue reasoning stream instrumentation\". Step 3: use the first web_search result url and call summarize_source. After summarize_source returns, reply with exactly PROMPT_DONE and no other text.", - "type": "text" - } - ], - "role": "user" - }, - { - "content": [ - { - "signature": "EoQFCm4IDhgCKkD+AoKMhpXNPUJ5eANwM28uUCJPCE+AXh0ACw7aXniX+YN04zdJl3hlHMbtAuKhhczLiEeOJtu4KtVpea5u5NFdMhpjbGF1ZGUtc29ubmV0LTQtNS0yMDI1MDkyOTgAQgh0aGlua2luZxIMIOTWCNEtsiI9vF4SGgwPXC861uZx8lpOziEiMLsCukGIluD0U44degzYnHFc+BwXNIEB+yatSHlZE+a5smr9crZcft3OLmsIdXuaFSrDAyQxWd9oqfJdnvza9eWIaOCeNkVCjrTc/wotBwh5Xp8m9k/WKt7P4zzUJBuLQ1f6ZnsdWkL8OiL/2E7Up55z2+UuVWsg1Y90/x27YRFdQjv3RnztYp4bZ5oIHAlTJoCzWiBYEh/Z8c2O+wy6MHo1cp0T57OroB/4HAqRZxrjTOXpKYLmYKXAY5tU8DTd3A0/FfEhg3oPSbOhX+112szFCpm3BRNJYooUUhwSu1hO4H/6r7d/Yp9XIM4tBfzuDx83CC/YetdOFgnYkdliw0xaUGaj4o37rWIRctWHAMdlUc4NDA0Ljc1uirHDXkN8EJwWSoIjPSBTA/p/wgLrvRV5Yy9Z0dMoWOpgYF6t1H/tNAyY8ml1CA7IWyd3jnfTZ1/G3Y0xFDwJDFlb4/QKc67w36SSCa+X9mMoTxD/DYN249lsII+XX3lQ7F1zJ07THOtCyRirCaRhmE/kGwl5tiB1QHrWa9bNHxQxf9i92435JGKN053Fe14AR70MtBSzIU2/zbgvhB1vF5lZt2i46hNxfo4lXR495gEwdwC6+8HJnlEJAJkdeHfZKj+RiwXoiYKPKsSbHi83X8Hh32IifuglNxdlxVQYAQ==", - "thinking": "The user wants me to complete a research flow with exactly three steps:\n1. Call lookup with query \"flue instrumentation\"\n2. Call web_search with the lookupId from step 1 and query \"Braintrust Flue reasoning stream instrumentation\"\n3. Call summarize_source with the first URL from step 2\n4. Reply with exactly \"PROMPT_DONE\"\n\nThe user emphasizes I should call exactly one tool per turn and wait for each result. Let me start with step 1.", - "type": "thinking" - }, - { - "text": "I'll complete this instrumented research flow step by step. Starting with Step 1:", - "type": "text" - }, - { - "id": "toolu_01MbotvqKgQV4ecxQ566vyuC", - "input": { - "query": "flue instrumentation" - }, - "name": "lookup", - "type": "tool_use" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "content": "{\"id\":\"flue-session-2026\",\"query\":\"flue instrumentation\",\"topic\":\"session instrumentation\"}", - "is_error": false, - "tool_use_id": "toolu_01MbotvqKgQV4ecxQ566vyuC", - "type": "tool_result" - } - ], - "role": "user" - }, - { - "content": [ - { - "signature": "EpEDCm4IDhgCKkAMahve1kB1j4njp1ZSzO8eCK4bb5Ex7MB6tjdoOXfuUDRfkBqvIGdwVly8GiA8GGYC7Y/RVfREXF6A6OXHTwFyMhpjbGF1ZGUtc29ubmV0LTQtNS0yMDI1MDkyOTgAQgh0aGlua2luZxIMLw+fsa4yo7a7ZSoIGgyaTdvH020n2dQj9bUiMB4OokY29w+uZb4rqDnKDbLXUqUj6U0f1QCGQF2ZBYDDfDJyM5Jv1xY3rQiUUjkhBirQAS4XivjvMnWI5zHBmpI/u/pq4aGdrdb3vUsJ/ne6R1d80JA/INGF8vLkQcZ5xxEK0jU4yRE+ldc2uzavmtoxIKAFD6bqKks1Qv6u+TIZtuW7tUB/7jFxDw93KmJNruMIa8c1J6pPUyBy3xE7QwgmU0kDfIfdnljxLoEBiMwtnxjCPj9lT7HHz4FkrhbnQTV4MvAQohCLjaSbOTd6LdLxwtJPm9mc26ID4eYN1gCCzyAxGXhqLHlzc23PafvQjsZliWnLhaiNMnG0YEo78XkuJQsYAQ==", - "thinking": "Good, I got the lookup result with id \"flue-session-2026\". Now I need to proceed to step 2: call web_search with this lookupId and the query \"Braintrust Flue reasoning stream instrumentation\".", - "type": "thinking" - }, - { - "text": "Now proceeding to Step 2:", - "type": "text" - }, - { - "id": "toolu_018rc1ig7vXeron3rLfpUtRT", - "input": { - "lookupId": "flue-session-2026", - "query": "Braintrust Flue reasoning stream instrumentation" - }, - "name": "web_search", - "type": "tool_use" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "cache_control": { - "type": "ephemeral" - }, - "content": "{\"lookupId\":\"flue-session-2026\",\"query\":\"Braintrust Flue reasoning stream instrumentation\",\"results\":[{\"title\":\"Flue reasoning stream instrumentation\",\"url\":\"https://example.test/flue/reasoning-streams\"}]}", - "is_error": false, - "tool_use_id": "toolu_018rc1ig7vXeron3rLfpUtRT", - "type": "tool_result" - } - ], - "role": "user" - } - ], - "model": "claude-sonnet-4-5-20250929", - "stream": true, - "system": [ - { - "cache_control": { - "type": "ephemeral" - }, - "text": "You are running in headless mode with no human operator. Work autonomously — never ask questions, never wait for user input. Make your best judgment and proceed independently.\n\nYou are a deterministic Flue instrumentation test agent. Follow user instructions exactly. When asked for a marker, output only that marker and no extra text. When running a local skill file, read it yourself and do not delegate it to a task.\n\n## Available Skills\n\nThe following skills provide specialized instructions for specific tasks. When a task matches a skill description, activate that skill before proceeding so its full instructions are loaded. Skill instructions and supporting resources stay lazy until activation or explicit file reads.\n\n- **e2e-flue-skill** — Deterministic skill used by the Braintrust Flue instrumentation e2e test.\n\nDate: Thu, May 28, 2026\nWorking directory: /Users/lucaforstner/conductor/workspaces/braintrust-sdk-javascript/shanghai-v1/e2e/.bt-tmp/run-u9uXnv/scenarios/flue-instrumentation\n\nDirectory structure:\n.agents\n__cassettes__\n__snapshots__\nassertions.ts\ncassette-filter.mjs\nconstants.mjs\nnode_modules\npackage.json\npnpm-lock.yaml\nscenario.impl.mjs\nscenario.mjs\nscenario.test.ts\nscenario.ts", - "type": "text" - } - ], - "thinking": { - "budget_tokens": 8192, - "display": "summarized", - "type": "enabled" - }, - "tools": [ - { - "description": "Read a file or list a directory. For files, output is truncated to 2000 lines or 50KB — use offset/limit for large files. For directories, returns the list of entries.", - "eager_input_streaming": true, - "input_schema": { - "properties": { - "limit": { - "description": "Maximum number of lines to read", - "type": "number" - }, - "offset": { - "description": "Line number to start from (1-indexed)", - "type": "number" - }, - "path": { - "description": "Path to the file to read", - "type": "string" - } - }, - "required": ["path"], - "type": "object" - }, - "name": "read" - }, - { - "description": "Write content to a file. Creates the file and parent directories if they do not exist.", - "eager_input_streaming": true, - "input_schema": { - "properties": { - "content": { - "description": "Content to write to the file", - "type": "string" - }, - "path": { - "description": "Path to the file to write", - "type": "string" - } - }, - "required": ["path", "content"], - "type": "object" - }, - "name": "write" - }, - { - "description": "Edit a file using exact text replacement. The oldText must match a unique region of the file. Use replaceAll to replace all occurrences.", - "eager_input_streaming": true, - "input_schema": { - "properties": { - "newText": { - "description": "Replacement text", - "type": "string" - }, - "oldText": { - "description": "Exact text to find (must be unique)", - "type": "string" - }, - "path": { - "description": "Path to the file to edit", - "type": "string" - }, - "replaceAll": { - "description": "Replace all occurrences", - "type": "boolean" - } - }, - "required": ["path", "oldText", "newText"], - "type": "object" - }, - "name": "edit" - }, - { - "description": "Execute a bash command. Returns stdout and stderr. Output is truncated to the last 2000 lines or 50KB.", - "eager_input_streaming": true, - "input_schema": { - "properties": { - "command": { - "description": "Bash command to execute", - "type": "string" - }, - "timeout": { - "description": "Timeout in seconds", - "type": "number" - } - }, - "required": ["command"], - "type": "object" - }, - "name": "bash" - }, - { - "description": "Search file contents for a regex pattern. Returns matching lines with file paths and line numbers.", - "eager_input_streaming": true, - "input_schema": { - "properties": { - "include": { - "description": "Glob filter, e.g. \"*.ts\"", - "type": "string" - }, - "path": { - "description": "Directory or file to search (default: .)", - "type": "string" - }, - "pattern": { - "description": "Search pattern (regex)", - "type": "string" - } - }, - "required": ["pattern"], - "type": "object" - }, - "name": "grep" - }, - { - "description": "Find files by filename pattern using shell find -name semantics. Returns matching file paths.", - "eager_input_streaming": true, - "input_schema": { - "properties": { - "path": { - "description": "Directory to search in (default: .)", - "type": "string" - }, - "pattern": { - "description": "Filename pattern, e.g. \"*.ts\"", - "type": "string" - } - }, - "required": ["pattern"], - "type": "object" - }, - "name": "glob" - }, - { - "description": "Delegate a focused task to a detached child agent with its own context. Use this for independent research, file exploration, or parallel work. The task returns only its final answer to this conversation. No subagents are currently defined.", - "eager_input_streaming": true, - "input_schema": { - "properties": { - "agent": { - "description": "Declared subagent to use for the child agent", - "type": "string" - }, - "cwd": { - "description": "Working directory for the child agent. AGENTS.md and skills are discovered from here.", - "type": "string" - }, - "description": { - "description": "Short human-readable label for the delegated work", - "type": "string" - }, - "prompt": { - "description": "Focused instructions for the child agent", - "type": "string" - } - }, - "required": ["prompt"], - "type": "object" - }, - "name": "task" - }, - { - "description": "Return a deterministic lookup result with an id needed by web_search.", - "eager_input_streaming": true, - "input_schema": { - "properties": { - "query": { - "type": "string" - } - }, - "required": ["query"], - "type": "object" - }, - "name": "lookup" - }, - { - "description": "Search a deterministic local web index. Requires the id returned by lookup.", - "eager_input_streaming": true, - "input_schema": { - "properties": { - "lookupId": { - "type": "string" - }, - "query": { - "type": "string" - } - }, - "required": ["lookupId", "query"], - "type": "object" - }, - "name": "web_search" - }, - { - "cache_control": { - "type": "ephemeral" - }, - "description": "Summarize the selected deterministic source after web_search returns a URL.", - "eager_input_streaming": true, - "input_schema": { - "properties": { - "url": { - "type": "string" - } - }, - "required": ["url"], - "type": "object" - }, - "name": "summarize_source" - } - ] - } - }, - "headers": {}, - "method": "POST", - "url": "https://api.anthropic.com/v1/messages" - }, - "response": { - "body": { - "chunks": [ - "event: message_start\ndata: {\"type\":\"message_start\",\"message\":{\"model\":\"claude-sonnet-4-5-20250929\",\"id\":\"msg_01HJoo9ncrvNPVcaq2UWN45L\",\"type\":\"message\",\"role\":\"assistant\",\"content\":[],\"stop_reason\":null,\"stop_sequence\":null,\"stop_details\":null,\"usage\":{\"input_tokens\":13,\"cache_creation_input_tokens\":234,\"cache_read_input_tokens\":2458,\"cache_creation\":{\"ephemeral_5m_input_tokens\":234,\"ephemeral_1h_input_tokens\":0},\"output_tokens\":4,\"service_tier\":\"standard\",\"inference_geo\":\"not_available\"}}}", - "event: content_block_start\ndata: {\"type\":\"content_block_start\",\"index\":0,\"content_block\":{\"type\":\"thinking\",\"thinking\":\"\",\"signature\":\"\"}}", - "event: ping\ndata: {\"type\":\"ping\"}", - "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"thinking_delta\",\"thinking\":\"Perfect! I got\"}}", - "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"thinking_delta\",\"thinking\":\" the web_search results with one result. The first (and only) result has the URL\"}}", - "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"thinking_delta\",\"thinking\":\" \\\"https://example.test/flue/reasoning-streams\\\". Now I need to proceed to step 3: call summarize_source with this URL.\"}}", - "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"signature_delta\",\"signature\":\"EqQDCm4IDhgCKkAXv7EDAhWMcrBz11bUd5qxlzyy03CAGQfRgLHDE0uKzLATh1o2sedd8GxLwwQqV0CmLRfNxsdK/ZDp3Lks9iysMhpjbGF1ZGUtc29ubmV0LTQtNS0yMDI1MDkyOTgAQgh0aGlua2luZxIMXDMniOy5Dy0KI8x7GgzgHs7BYJMs6/N0ur8iMPe0SwnHhP39OjEu7P8ASwK9ffwa3VP67QOlPzOstRRdEZNd0mq+BVjpmBXB+QcAzCrjAeAh/UO/d7SfFmXjqwA3//thoL/TTlGa6W+fdOCFRikn65JWT4ZdVFetfqD+9/QCP/nXEwMsLnTBAxC4EGryrgs2YT+OWwmEJ7rHKFeDs5djhwSb2J2ZJO1kRvD9s2vCCDkImygYBSIVgx1Y0CZJHhzn2Vf39v0Fbth2wDZa+OqxivV9NmDZEjmgcM+tTRx1CCnaww8LZEJQ68v/dUJy7hhD4MrNcQ62evJYPfT3+4EKKWJ/7HI7KXz91//+YAkph63XstbJMFJ3AldAFiq3ecauXt5r42N5gmCmwZX+sI48ENLGGAE=\"}}", - "event: content_block_stop\ndata: {\"type\":\"content_block_stop\",\"index\":0}", - "event: content_block_start\ndata: {\"type\":\"content_block_start\",\"index\":1,\"content_block\":{\"type\":\"text\",\"text\":\"\"}}", - "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":1,\"delta\":{\"type\":\"text_delta\",\"text\":\"Now proceeding to Step 3:\"}}", - "event: content_block_stop\ndata: {\"type\":\"content_block_stop\",\"index\":1}", - "event: content_block_start\ndata: {\"type\":\"content_block_start\",\"index\":2,\"content_block\":{\"type\":\"tool_use\",\"id\":\"toolu_01MRjt3UmRqrUYHt1ZC8gND9\",\"name\":\"summarize_source\",\"input\":{},\"caller\":{\"type\":\"direct\"}}}", - "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":2,\"delta\":{\"type\":\"input_json_delta\",\"partial_json\":\"\"}}", - "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":2,\"delta\":{\"type\":\"input_json_delta\",\"partial_json\":\"{\\\"url\\\": \\\"https://example.test/flue/reasoning-streams\"}}", - "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":2,\"delta\":{\"type\":\"input_json_delta\",\"partial_json\":\"\\\"}\"}}", - "event: content_block_stop\ndata: {\"type\":\"content_block_stop\",\"index\":2}", - "event: message_delta\ndata: {\"type\":\"message_delta\",\"delta\":{\"stop_reason\":\"tool_use\",\"stop_sequence\":null,\"stop_details\":null},\"usage\":{\"input_tokens\":13,\"cache_creation_input_tokens\":234,\"cache_read_input_tokens\":2458,\"output_tokens\":138,\"output_tokens_details\":{\"thinking_tokens\":61}}}", - "event: message_stop\ndata: {\"type\":\"message_stop\"}" - ], - "kind": "sse" - }, - "headers": { - "anthropic-organization-id": "27796668-7351-40ac-acc4-024aee8995a5", - "anthropic-ratelimit-input-tokens-limit": "3000000", - "anthropic-ratelimit-input-tokens-remaining": "2998000", - "anthropic-ratelimit-input-tokens-reset": "2026-05-28T13:40:54Z", - "anthropic-ratelimit-output-tokens-limit": "600000", - "anthropic-ratelimit-output-tokens-remaining": "600000", - "anthropic-ratelimit-output-tokens-reset": "2026-05-28T13:40:54Z", - "anthropic-ratelimit-requests-limit": "20000", - "anthropic-ratelimit-requests-remaining": "19999", - "anthropic-ratelimit-requests-reset": "2026-05-28T13:40:54Z", - "anthropic-ratelimit-tokens-limit": "3600000", - "anthropic-ratelimit-tokens-remaining": "3598000", - "anthropic-ratelimit-tokens-reset": "2026-05-28T13:40:54Z", - "cache-control": "no-cache", - "cf-cache-status": "DYNAMIC", - "cf-ray": "a02dac7d4a1fc301-VIE", - "connection": "keep-alive", - "content-encoding": "gzip", - "content-security-policy": "default-src 'none'; frame-ancestors 'none'", - "content-type": "text/event-stream; charset=utf-8", - "date": "Thu, 28 May 2026 13:40:56 GMT", - "request-id": "req_011CbV4ZUiiFpwboN3YtdzkH", - "server": "cloudflare", - "set-cookie": "[REDACTED]", - "strict-transport-security": "max-age=31536000; includeSubDomains; preload", - "traceresponse": "00-6392939f911bfe9568c361c78254c370-cb7c6141cf28bb77-01", - "transfer-encoding": "chunked", - "vary": "Accept-Encoding", - "x-envoy-upstream-service-time": "2757", - "x-robots-tag": "none" - }, - "status": 200, - "statusText": "OK" - } - }, - { - "callIndex": 3, - "id": "305957222aa1f6e9", - "matchKey": "POST api.anthropic.com/v1/messages", - "recordedAt": "2026-05-28T13:41:01.029Z", - "request": { - "body": { - "kind": "json", - "value": { - "max_tokens": 64000, - "messages": [ - { - "content": [ - { - "text": "Complete this instrumented research flow. Call exactly one tool per turn and wait for each tool result before choosing the next tool. Step 1: call lookup with query \"flue instrumentation\". Step 2: use the lookup result id as lookupId and call web_search with query \"Braintrust Flue reasoning stream instrumentation\". Step 3: use the first web_search result url and call summarize_source. After summarize_source returns, reply with exactly PROMPT_DONE and no other text.", - "type": "text" - } - ], - "role": "user" - }, - { - "content": [ - { - "signature": "EoQFCm4IDhgCKkD+AoKMhpXNPUJ5eANwM28uUCJPCE+AXh0ACw7aXniX+YN04zdJl3hlHMbtAuKhhczLiEeOJtu4KtVpea5u5NFdMhpjbGF1ZGUtc29ubmV0LTQtNS0yMDI1MDkyOTgAQgh0aGlua2luZxIMIOTWCNEtsiI9vF4SGgwPXC861uZx8lpOziEiMLsCukGIluD0U44degzYnHFc+BwXNIEB+yatSHlZE+a5smr9crZcft3OLmsIdXuaFSrDAyQxWd9oqfJdnvza9eWIaOCeNkVCjrTc/wotBwh5Xp8m9k/WKt7P4zzUJBuLQ1f6ZnsdWkL8OiL/2E7Up55z2+UuVWsg1Y90/x27YRFdQjv3RnztYp4bZ5oIHAlTJoCzWiBYEh/Z8c2O+wy6MHo1cp0T57OroB/4HAqRZxrjTOXpKYLmYKXAY5tU8DTd3A0/FfEhg3oPSbOhX+112szFCpm3BRNJYooUUhwSu1hO4H/6r7d/Yp9XIM4tBfzuDx83CC/YetdOFgnYkdliw0xaUGaj4o37rWIRctWHAMdlUc4NDA0Ljc1uirHDXkN8EJwWSoIjPSBTA/p/wgLrvRV5Yy9Z0dMoWOpgYF6t1H/tNAyY8ml1CA7IWyd3jnfTZ1/G3Y0xFDwJDFlb4/QKc67w36SSCa+X9mMoTxD/DYN249lsII+XX3lQ7F1zJ07THOtCyRirCaRhmE/kGwl5tiB1QHrWa9bNHxQxf9i92435JGKN053Fe14AR70MtBSzIU2/zbgvhB1vF5lZt2i46hNxfo4lXR495gEwdwC6+8HJnlEJAJkdeHfZKj+RiwXoiYKPKsSbHi83X8Hh32IifuglNxdlxVQYAQ==", - "thinking": "The user wants me to complete a research flow with exactly three steps:\n1. Call lookup with query \"flue instrumentation\"\n2. Call web_search with the lookupId from step 1 and query \"Braintrust Flue reasoning stream instrumentation\"\n3. Call summarize_source with the first URL from step 2\n4. Reply with exactly \"PROMPT_DONE\"\n\nThe user emphasizes I should call exactly one tool per turn and wait for each result. Let me start with step 1.", - "type": "thinking" - }, - { - "text": "I'll complete this instrumented research flow step by step. Starting with Step 1:", - "type": "text" - }, - { - "id": "toolu_01MbotvqKgQV4ecxQ566vyuC", - "input": { - "query": "flue instrumentation" - }, - "name": "lookup", - "type": "tool_use" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "content": "{\"id\":\"flue-session-2026\",\"query\":\"flue instrumentation\",\"topic\":\"session instrumentation\"}", - "is_error": false, - "tool_use_id": "toolu_01MbotvqKgQV4ecxQ566vyuC", - "type": "tool_result" - } - ], - "role": "user" - }, - { - "content": [ - { - "signature": "EpEDCm4IDhgCKkAMahve1kB1j4njp1ZSzO8eCK4bb5Ex7MB6tjdoOXfuUDRfkBqvIGdwVly8GiA8GGYC7Y/RVfREXF6A6OXHTwFyMhpjbGF1ZGUtc29ubmV0LTQtNS0yMDI1MDkyOTgAQgh0aGlua2luZxIMLw+fsa4yo7a7ZSoIGgyaTdvH020n2dQj9bUiMB4OokY29w+uZb4rqDnKDbLXUqUj6U0f1QCGQF2ZBYDDfDJyM5Jv1xY3rQiUUjkhBirQAS4XivjvMnWI5zHBmpI/u/pq4aGdrdb3vUsJ/ne6R1d80JA/INGF8vLkQcZ5xxEK0jU4yRE+ldc2uzavmtoxIKAFD6bqKks1Qv6u+TIZtuW7tUB/7jFxDw93KmJNruMIa8c1J6pPUyBy3xE7QwgmU0kDfIfdnljxLoEBiMwtnxjCPj9lT7HHz4FkrhbnQTV4MvAQohCLjaSbOTd6LdLxwtJPm9mc26ID4eYN1gCCzyAxGXhqLHlzc23PafvQjsZliWnLhaiNMnG0YEo78XkuJQsYAQ==", - "thinking": "Good, I got the lookup result with id \"flue-session-2026\". Now I need to proceed to step 2: call web_search with this lookupId and the query \"Braintrust Flue reasoning stream instrumentation\".", - "type": "thinking" - }, - { - "text": "Now proceeding to Step 2:", - "type": "text" - }, - { - "id": "toolu_018rc1ig7vXeron3rLfpUtRT", - "input": { - "lookupId": "flue-session-2026", - "query": "Braintrust Flue reasoning stream instrumentation" - }, - "name": "web_search", - "type": "tool_use" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "content": "{\"lookupId\":\"flue-session-2026\",\"query\":\"Braintrust Flue reasoning stream instrumentation\",\"results\":[{\"title\":\"Flue reasoning stream instrumentation\",\"url\":\"https://example.test/flue/reasoning-streams\"}]}", - "is_error": false, - "tool_use_id": "toolu_018rc1ig7vXeron3rLfpUtRT", - "type": "tool_result" - } - ], - "role": "user" - }, - { - "content": [ - { - "signature": "EqQDCm4IDhgCKkAXv7EDAhWMcrBz11bUd5qxlzyy03CAGQfRgLHDE0uKzLATh1o2sedd8GxLwwQqV0CmLRfNxsdK/ZDp3Lks9iysMhpjbGF1ZGUtc29ubmV0LTQtNS0yMDI1MDkyOTgAQgh0aGlua2luZxIMXDMniOy5Dy0KI8x7GgzgHs7BYJMs6/N0ur8iMPe0SwnHhP39OjEu7P8ASwK9ffwa3VP67QOlPzOstRRdEZNd0mq+BVjpmBXB+QcAzCrjAeAh/UO/d7SfFmXjqwA3//thoL/TTlGa6W+fdOCFRikn65JWT4ZdVFetfqD+9/QCP/nXEwMsLnTBAxC4EGryrgs2YT+OWwmEJ7rHKFeDs5djhwSb2J2ZJO1kRvD9s2vCCDkImygYBSIVgx1Y0CZJHhzn2Vf39v0Fbth2wDZa+OqxivV9NmDZEjmgcM+tTRx1CCnaww8LZEJQ68v/dUJy7hhD4MrNcQ62evJYPfT3+4EKKWJ/7HI7KXz91//+YAkph63XstbJMFJ3AldAFiq3ecauXt5r42N5gmCmwZX+sI48ENLGGAE=", - "thinking": "Perfect! I got the web_search results with one result. The first (and only) result has the URL \"https://example.test/flue/reasoning-streams\". Now I need to proceed to step 3: call summarize_source with this URL.", - "type": "thinking" - }, - { - "text": "Now proceeding to Step 3:", - "type": "text" - }, - { - "id": "toolu_01MRjt3UmRqrUYHt1ZC8gND9", - "input": { - "url": "https://example.test/flue/reasoning-streams" - }, - "name": "summarize_source", - "type": "tool_use" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "cache_control": { - "type": "ephemeral" - }, - "content": "{\"summary\":\"Flue emits reasoning, tool execution, and LLM turn events separately.\",\"url\":\"https://example.test/flue/reasoning-streams\"}", - "is_error": false, - "tool_use_id": "toolu_01MRjt3UmRqrUYHt1ZC8gND9", - "type": "tool_result" - } - ], - "role": "user" - } - ], - "model": "claude-sonnet-4-5-20250929", - "stream": true, - "system": [ - { - "cache_control": { - "type": "ephemeral" - }, - "text": "You are running in headless mode with no human operator. Work autonomously — never ask questions, never wait for user input. Make your best judgment and proceed independently.\n\nYou are a deterministic Flue instrumentation test agent. Follow user instructions exactly. When asked for a marker, output only that marker and no extra text. When running a local skill file, read it yourself and do not delegate it to a task.\n\n## Available Skills\n\nThe following skills provide specialized instructions for specific tasks. When a task matches a skill description, activate that skill before proceeding so its full instructions are loaded. Skill instructions and supporting resources stay lazy until activation or explicit file reads.\n\n- **e2e-flue-skill** — Deterministic skill used by the Braintrust Flue instrumentation e2e test.\n\nDate: Thu, May 28, 2026\nWorking directory: /Users/lucaforstner/conductor/workspaces/braintrust-sdk-javascript/shanghai-v1/e2e/.bt-tmp/run-u9uXnv/scenarios/flue-instrumentation\n\nDirectory structure:\n.agents\n__cassettes__\n__snapshots__\nassertions.ts\ncassette-filter.mjs\nconstants.mjs\nnode_modules\npackage.json\npnpm-lock.yaml\nscenario.impl.mjs\nscenario.mjs\nscenario.test.ts\nscenario.ts", - "type": "text" - } - ], - "thinking": { - "budget_tokens": 8192, - "display": "summarized", - "type": "enabled" - }, - "tools": [ - { - "description": "Read a file or list a directory. For files, output is truncated to 2000 lines or 50KB — use offset/limit for large files. For directories, returns the list of entries.", - "eager_input_streaming": true, - "input_schema": { - "properties": { - "limit": { - "description": "Maximum number of lines to read", - "type": "number" - }, - "offset": { - "description": "Line number to start from (1-indexed)", - "type": "number" - }, - "path": { - "description": "Path to the file to read", - "type": "string" - } - }, - "required": ["path"], - "type": "object" - }, - "name": "read" - }, - { - "description": "Write content to a file. Creates the file and parent directories if they do not exist.", - "eager_input_streaming": true, - "input_schema": { - "properties": { - "content": { - "description": "Content to write to the file", - "type": "string" - }, - "path": { - "description": "Path to the file to write", - "type": "string" - } - }, - "required": ["path", "content"], - "type": "object" - }, - "name": "write" - }, - { - "description": "Edit a file using exact text replacement. The oldText must match a unique region of the file. Use replaceAll to replace all occurrences.", - "eager_input_streaming": true, - "input_schema": { - "properties": { - "newText": { - "description": "Replacement text", - "type": "string" - }, - "oldText": { - "description": "Exact text to find (must be unique)", - "type": "string" - }, - "path": { - "description": "Path to the file to edit", - "type": "string" - }, - "replaceAll": { - "description": "Replace all occurrences", - "type": "boolean" - } - }, - "required": ["path", "oldText", "newText"], - "type": "object" - }, - "name": "edit" - }, - { - "description": "Execute a bash command. Returns stdout and stderr. Output is truncated to the last 2000 lines or 50KB.", - "eager_input_streaming": true, - "input_schema": { - "properties": { - "command": { - "description": "Bash command to execute", - "type": "string" - }, - "timeout": { - "description": "Timeout in seconds", - "type": "number" - } - }, - "required": ["command"], - "type": "object" - }, - "name": "bash" - }, - { - "description": "Search file contents for a regex pattern. Returns matching lines with file paths and line numbers.", - "eager_input_streaming": true, - "input_schema": { - "properties": { - "include": { - "description": "Glob filter, e.g. \"*.ts\"", - "type": "string" - }, - "path": { - "description": "Directory or file to search (default: .)", - "type": "string" - }, - "pattern": { - "description": "Search pattern (regex)", - "type": "string" - } - }, - "required": ["pattern"], - "type": "object" - }, - "name": "grep" - }, - { - "description": "Find files by filename pattern using shell find -name semantics. Returns matching file paths.", - "eager_input_streaming": true, - "input_schema": { - "properties": { - "path": { - "description": "Directory to search in (default: .)", - "type": "string" - }, - "pattern": { - "description": "Filename pattern, e.g. \"*.ts\"", - "type": "string" - } - }, - "required": ["pattern"], - "type": "object" - }, - "name": "glob" - }, - { - "description": "Delegate a focused task to a detached child agent with its own context. Use this for independent research, file exploration, or parallel work. The task returns only its final answer to this conversation. No subagents are currently defined.", - "eager_input_streaming": true, - "input_schema": { - "properties": { - "agent": { - "description": "Declared subagent to use for the child agent", - "type": "string" - }, - "cwd": { - "description": "Working directory for the child agent. AGENTS.md and skills are discovered from here.", - "type": "string" - }, - "description": { - "description": "Short human-readable label for the delegated work", - "type": "string" - }, - "prompt": { - "description": "Focused instructions for the child agent", - "type": "string" - } - }, - "required": ["prompt"], - "type": "object" - }, - "name": "task" - }, - { - "description": "Return a deterministic lookup result with an id needed by web_search.", - "eager_input_streaming": true, - "input_schema": { - "properties": { - "query": { - "type": "string" - } - }, - "required": ["query"], - "type": "object" - }, - "name": "lookup" - }, - { - "description": "Search a deterministic local web index. Requires the id returned by lookup.", - "eager_input_streaming": true, - "input_schema": { - "properties": { - "lookupId": { - "type": "string" - }, - "query": { - "type": "string" - } - }, - "required": ["lookupId", "query"], - "type": "object" - }, - "name": "web_search" - }, - { - "cache_control": { - "type": "ephemeral" - }, - "description": "Summarize the selected deterministic source after web_search returns a URL.", - "eager_input_streaming": true, - "input_schema": { - "properties": { - "url": { - "type": "string" - } - }, - "required": ["url"], - "type": "object" - }, - "name": "summarize_source" - } - ] - } - }, - "headers": {}, - "method": "POST", - "url": "https://api.anthropic.com/v1/messages" - }, - "response": { - "body": { - "chunks": [ - "event: message_start\ndata: {\"type\":\"message_start\",\"message\":{\"model\":\"claude-sonnet-4-5-20250929\",\"id\":\"msg_019YC1YEtmEwfw9dyuY3MPdx\",\"type\":\"message\",\"role\":\"assistant\",\"content\":[],\"stop_reason\":null,\"stop_sequence\":null,\"stop_details\":null,\"usage\":{\"input_tokens\":13,\"cache_creation_input_tokens\":194,\"cache_read_input_tokens\":2692,\"cache_creation\":{\"ephemeral_5m_input_tokens\":194,\"ephemeral_1h_input_tokens\":0},\"output_tokens\":1,\"service_tier\":\"standard\",\"inference_geo\":\"not_available\"}}}", - "event: content_block_start\ndata: {\"type\":\"content_block_start\",\"index\":0,\"content_block\":{\"type\":\"thinking\",\"thinking\":\"\",\"signature\":\"\"}}", - "event: ping\ndata: {\"type\":\"ping\"}", - "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"thinking_delta\",\"thinking\":\"Perfect\"}}", - "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"thinking_delta\",\"thinking\":\"! The summarize_source call has completed successfully. According to the instructions, I should now reply with exactly \\\"PROMPT_DONE\\\" and no other text.\"}}", - "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"thinking_delta\",\"thinking\":\"\"}}", - "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"signature_delta\",\"signature\":\"Eu8CCm4IDhgCKkAn2QOIIlYfvcs6ynsmhCslHDtBsrTA4CfmimK7zyz6+k6dZldV2JBtEBET41It0gFcGEKC0n0mAhaYVuLutpDDMhpjbGF1ZGUtc29ubmV0LTQtNS0yMDI1MDkyOTgAQgh0aGlua2luZxIM1xfLptxSF8rKzou8GgwRLuVmfiBYOT4CZIsiMAYz8TpvchMAWodP+T9uZ+dJ47MVdNv+p86nZXmT5d5EWlCroiAOFHHzJH2o0UjJQSquAX6wu+jIC8/DXwurGIIf2ToldPuJWF5HGj8kHMnwh777XS1NUFl1irM24rvP7kTAv3Dp8YEwb8t3g1EsicY7j+C5DzgwxVsPYCVOCgdMqQjtsqpL7AX5KQ9+hR1gK1DQjshv4Gze3hw/+C+9iptd3RlDIuwO0cm8NY+sVqeWVh3TtQujQlVt1rW/HfWjclrtT0LZSGwc6RypmfMMQi6lncIk/EPgy0XbS1plKW0tLBgB\"}}", - "event: content_block_stop\ndata: {\"type\":\"content_block_stop\",\"index\":0}", - "event: content_block_start\ndata: {\"type\":\"content_block_start\",\"index\":1,\"content_block\":{\"type\":\"text\",\"text\":\"\"}}", - "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":1,\"delta\":{\"type\":\"text_delta\",\"text\":\"PROMPT_DONE\"}}", - "event: content_block_stop\ndata: {\"type\":\"content_block_stop\",\"index\":1}", - "event: message_delta\ndata: {\"type\":\"message_delta\",\"delta\":{\"stop_reason\":\"end_turn\",\"stop_sequence\":null,\"stop_details\":null},\"usage\":{\"input_tokens\":13,\"cache_creation_input_tokens\":194,\"cache_read_input_tokens\":2692,\"output_tokens\":51,\"output_tokens_details\":{\"thinking_tokens\":40}}}", - "event: message_stop\ndata: {\"type\":\"message_stop\"}" - ], - "kind": "sse" - }, - "headers": { - "anthropic-organization-id": "27796668-7351-40ac-acc4-024aee8995a5", - "anthropic-ratelimit-input-tokens-limit": "3000000", - "anthropic-ratelimit-input-tokens-remaining": "2998000", - "anthropic-ratelimit-input-tokens-reset": "2026-05-28T13:40:58Z", - "anthropic-ratelimit-output-tokens-limit": "600000", - "anthropic-ratelimit-output-tokens-remaining": "600000", - "anthropic-ratelimit-output-tokens-reset": "2026-05-28T13:40:58Z", - "anthropic-ratelimit-requests-limit": "20000", - "anthropic-ratelimit-requests-remaining": "19999", - "anthropic-ratelimit-requests-reset": "2026-05-28T13:40:58Z", - "anthropic-ratelimit-tokens-limit": "3600000", - "anthropic-ratelimit-tokens-remaining": "3598000", - "anthropic-ratelimit-tokens-reset": "2026-05-28T13:40:58Z", - "cache-control": "no-cache", - "cf-cache-status": "DYNAMIC", - "cf-ray": "a02dac9afd8fc301-VIE", - "connection": "keep-alive", - "content-encoding": "gzip", - "content-security-policy": "default-src 'none'; frame-ancestors 'none'", - "content-type": "text/event-stream; charset=utf-8", - "date": "Thu, 28 May 2026 13:41:00 GMT", - "request-id": "req_011CbV4Zq3L7gJxP1Y46qapx", - "server": "cloudflare", - "set-cookie": "[REDACTED]", - "strict-transport-security": "max-age=31536000; includeSubDomains; preload", - "traceresponse": "00-e0b4a5e0f6b8f2625262dec81a52e61e-c551de5bb2f6f203-01", - "transfer-encoding": "chunked", - "vary": "Accept-Encoding", - "x-envoy-upstream-service-time": "1421", - "x-robots-tag": "none" - }, - "status": 200, - "statusText": "OK" - } - }, - { - "callIndex": 4, - "id": "3d93fe37f72a947f", - "matchKey": "POST api.anthropic.com/v1/messages", - "recordedAt": "2026-05-28T13:41:05.303Z", - "request": { - "body": { - "kind": "json", - "value": { - "max_tokens": 64000, - "messages": [ - { - "content": [ - { - "cache_control": { - "type": "ephemeral" - }, - "text": "Run the skill named \"e2e-flue-skill\".\n\nArguments:\n{\n \"marker\": \"SKILL_DONE\"\n}", - "type": "text" - } - ], - "role": "user" - } - ], - "model": "claude-sonnet-4-5-20250929", - "stream": true, - "system": [ - { - "cache_control": { - "type": "ephemeral" - }, - "text": "You are running in headless mode with no human operator. Work autonomously — never ask questions, never wait for user input. Make your best judgment and proceed independently.\n\nYou are a deterministic Flue instrumentation test agent. Follow user instructions exactly. When asked for a marker, output only that marker and no extra text. When running a local skill file, read it yourself and do not delegate it to a task.\n\n## Available Skills\n\nThe following skills provide specialized instructions for specific tasks. When a task matches a skill description, activate that skill before proceeding so its full instructions are loaded. Skill instructions and supporting resources stay lazy until activation or explicit file reads.\n\n- **e2e-flue-skill** — Deterministic skill used by the Braintrust Flue instrumentation e2e test.\n\nDate: Thu, May 28, 2026\nWorking directory: /Users/lucaforstner/conductor/workspaces/braintrust-sdk-javascript/shanghai-v1/e2e/.bt-tmp/run-u9uXnv/scenarios/flue-instrumentation\n\nDirectory structure:\n.agents\n__cassettes__\n__snapshots__\nassertions.ts\ncassette-filter.mjs\nconstants.mjs\nnode_modules\npackage.json\npnpm-lock.yaml\nscenario.impl.mjs\nscenario.mjs\nscenario.test.ts\nscenario.ts", - "type": "text" - } - ], - "thinking": { - "type": "disabled" - }, - "tools": [ - { - "description": "Read a file or list a directory. For files, output is truncated to 2000 lines or 50KB — use offset/limit for large files. For directories, returns the list of entries.", - "eager_input_streaming": true, - "input_schema": { - "properties": { - "limit": { - "description": "Maximum number of lines to read", - "type": "number" - }, - "offset": { - "description": "Line number to start from (1-indexed)", - "type": "number" - }, - "path": { - "description": "Path to the file to read", - "type": "string" - } - }, - "required": ["path"], - "type": "object" - }, - "name": "read" - }, - { - "description": "Write content to a file. Creates the file and parent directories if they do not exist.", - "eager_input_streaming": true, - "input_schema": { - "properties": { - "content": { - "description": "Content to write to the file", - "type": "string" - }, - "path": { - "description": "Path to the file to write", - "type": "string" - } - }, - "required": ["path", "content"], - "type": "object" - }, - "name": "write" - }, - { - "description": "Edit a file using exact text replacement. The oldText must match a unique region of the file. Use replaceAll to replace all occurrences.", - "eager_input_streaming": true, - "input_schema": { - "properties": { - "newText": { - "description": "Replacement text", - "type": "string" - }, - "oldText": { - "description": "Exact text to find (must be unique)", - "type": "string" - }, - "path": { - "description": "Path to the file to edit", - "type": "string" - }, - "replaceAll": { - "description": "Replace all occurrences", - "type": "boolean" - } - }, - "required": ["path", "oldText", "newText"], - "type": "object" - }, - "name": "edit" - }, - { - "description": "Execute a bash command. Returns stdout and stderr. Output is truncated to the last 2000 lines or 50KB.", - "eager_input_streaming": true, - "input_schema": { - "properties": { - "command": { - "description": "Bash command to execute", - "type": "string" - }, - "timeout": { - "description": "Timeout in seconds", - "type": "number" - } - }, - "required": ["command"], - "type": "object" - }, - "name": "bash" - }, - { - "description": "Search file contents for a regex pattern. Returns matching lines with file paths and line numbers.", - "eager_input_streaming": true, - "input_schema": { - "properties": { - "include": { - "description": "Glob filter, e.g. \"*.ts\"", - "type": "string" - }, - "path": { - "description": "Directory or file to search (default: .)", - "type": "string" - }, - "pattern": { - "description": "Search pattern (regex)", - "type": "string" - } - }, - "required": ["pattern"], - "type": "object" - }, - "name": "grep" - }, - { - "description": "Find files by filename pattern using shell find -name semantics. Returns matching file paths.", - "eager_input_streaming": true, - "input_schema": { - "properties": { - "path": { - "description": "Directory to search in (default: .)", - "type": "string" - }, - "pattern": { - "description": "Filename pattern, e.g. \"*.ts\"", - "type": "string" - } - }, - "required": ["pattern"], - "type": "object" - }, - "name": "glob" - }, - { - "cache_control": { - "type": "ephemeral" - }, - "description": "Delegate a focused task to a detached child agent with its own context. Use this for independent research, file exploration, or parallel work. The task returns only its final answer to this conversation. No subagents are currently defined.", - "eager_input_streaming": true, - "input_schema": { - "properties": { - "agent": { - "description": "Declared subagent to use for the child agent", - "type": "string" - }, - "cwd": { - "description": "Working directory for the child agent. AGENTS.md and skills are discovered from here.", - "type": "string" - }, - "description": { - "description": "Short human-readable label for the delegated work", - "type": "string" - }, - "prompt": { - "description": "Focused instructions for the child agent", - "type": "string" - } - }, - "required": ["prompt"], - "type": "object" - }, - "name": "task" - } - ] - } - }, - "headers": {}, - "method": "POST", - "url": "https://api.anthropic.com/v1/messages" - }, - "response": { - "body": { - "chunks": [ - "event: message_start\ndata: {\"type\":\"message_start\",\"message\":{\"model\":\"claude-sonnet-4-5-20250929\",\"id\":\"msg_01SoNm9ntxkXQkRrBSidrXSB\",\"type\":\"message\",\"role\":\"assistant\",\"content\":[],\"stop_reason\":null,\"stop_sequence\":null,\"stop_details\":null,\"usage\":{\"input_tokens\":3,\"cache_creation_input_tokens\":670,\"cache_read_input_tokens\":1099,\"cache_creation\":{\"ephemeral_5m_input_tokens\":670,\"ephemeral_1h_input_tokens\":0},\"output_tokens\":5,\"service_tier\":\"standard\",\"inference_geo\":\"not_available\"}}}", - "event: content_block_start\ndata: {\"type\":\"content_block_start\",\"index\":0,\"content_block\":{\"type\":\"text\",\"text\":\"\"}}", - "event: ping\ndata: {\"type\":\"ping\"}", - "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"text_delta\",\"text\":\"I'll run the e\"}}", - "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"text_delta\",\"text\":\"2e-flue-skill. Let me first read the skill file to understand its instructions.\"}}", - "event: content_block_stop\ndata: {\"type\":\"content_block_stop\",\"index\":0}", - "event: content_block_start\ndata: {\"type\":\"content_block_start\",\"index\":1,\"content_block\":{\"type\":\"tool_use\",\"id\":\"toolu_0116y5v4MmMkEaDkjkEPPjTH\",\"name\":\"read\",\"input\":{},\"caller\":{\"type\":\"direct\"}}}", - "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":1,\"delta\":{\"type\":\"input_json_delta\",\"partial_json\":\"\"}}", - "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":1,\"delta\":{\"type\":\"input_json_delta\",\"partial_json\":\"{\\\"path\\\": \\\".agents/e2e-flue\"}}", - "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":1,\"delta\":{\"type\":\"input_json_delta\",\"partial_json\":\"-skill.md\"}}", - "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":1,\"delta\":{\"type\":\"input_json_delta\",\"partial_json\":\"\\\"}\"}}", - "event: content_block_stop\ndata: {\"type\":\"content_block_stop\",\"index\":1}", - "event: message_delta\ndata: {\"type\":\"message_delta\",\"delta\":{\"stop_reason\":\"tool_use\",\"stop_sequence\":null,\"stop_details\":null},\"usage\":{\"input_tokens\":3,\"cache_creation_input_tokens\":670,\"cache_read_input_tokens\":1099,\"output_tokens\":89}}", - "event: message_stop\ndata: {\"type\":\"message_stop\"}" - ], - "kind": "sse" - }, - "headers": { - "anthropic-organization-id": "27796668-7351-40ac-acc4-024aee8995a5", - "anthropic-ratelimit-input-tokens-limit": "3000000", - "anthropic-ratelimit-input-tokens-remaining": "2999000", - "anthropic-ratelimit-input-tokens-reset": "2026-05-28T13:41:01Z", - "anthropic-ratelimit-output-tokens-limit": "600000", - "anthropic-ratelimit-output-tokens-remaining": "600000", - "anthropic-ratelimit-output-tokens-reset": "2026-05-28T13:41:01Z", - "anthropic-ratelimit-requests-limit": "20000", - "anthropic-ratelimit-requests-remaining": "19999", - "anthropic-ratelimit-requests-reset": "2026-05-28T13:41:01Z", - "anthropic-ratelimit-tokens-limit": "3600000", - "anthropic-ratelimit-tokens-remaining": "3599000", - "anthropic-ratelimit-tokens-reset": "2026-05-28T13:41:01Z", - "cache-control": "no-cache", - "cf-cache-status": "DYNAMIC", - "cf-ray": "a02dacab0e1fc301-VIE", - "connection": "keep-alive", - "content-encoding": "gzip", - "content-security-policy": "default-src 'none'; frame-ancestors 'none'", - "content-type": "text/event-stream; charset=utf-8", - "date": "Thu, 28 May 2026 13:41:04 GMT", - "request-id": "req_011CbV4a2cMY5XByBTCzUoHh", - "server": "cloudflare", - "set-cookie": "[REDACTED]", - "strict-transport-security": "max-age=31536000; includeSubDomains; preload", - "traceresponse": "00-e05c9702b7456a4b1c655ed13d92ba42-b48b861130669b0c-01", - "transfer-encoding": "chunked", - "vary": "Accept-Encoding", - "x-envoy-upstream-service-time": "2864", - "x-robots-tag": "none" - }, - "status": 200, - "statusText": "OK" - } - }, - { - "callIndex": 5, - "id": "fb4ed2bb1569d980", - "matchKey": "POST api.anthropic.com/v1/messages", - "recordedAt": "2026-05-28T13:41:08.068Z", - "request": { - "body": { - "kind": "json", - "value": { - "max_tokens": 64000, - "messages": [ - { - "content": [ - { - "text": "Run the skill named \"e2e-flue-skill\".\n\nArguments:\n{\n \"marker\": \"SKILL_DONE\"\n}", - "type": "text" - } - ], - "role": "user" - }, - { - "content": [ - { - "text": "I'll run the e2e-flue-skill. Let me first read the skill file to understand its instructions.", - "type": "text" - }, - { - "id": "toolu_0116y5v4MmMkEaDkjkEPPjTH", - "input": { - "path": ".agents/e2e-flue-skill.md" - }, - "name": "read", - "type": "tool_use" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "cache_control": { - "type": "ephemeral" - }, - "content": "ENOENT: no such file or directory, open '/Users/lucaforstner/conductor/workspaces/braintrust-sdk-javascript/shanghai-v1/e2e/.bt-tmp/run-u9uXnv/scenarios/flue-instrumentation/.agents/e2e-flue-skill.md'", - "is_error": true, - "tool_use_id": "toolu_0116y5v4MmMkEaDkjkEPPjTH", - "type": "tool_result" - } - ], - "role": "user" - } - ], - "model": "claude-sonnet-4-5-20250929", - "stream": true, - "system": [ - { - "cache_control": { - "type": "ephemeral" - }, - "text": "You are running in headless mode with no human operator. Work autonomously — never ask questions, never wait for user input. Make your best judgment and proceed independently.\n\nYou are a deterministic Flue instrumentation test agent. Follow user instructions exactly. When asked for a marker, output only that marker and no extra text. When running a local skill file, read it yourself and do not delegate it to a task.\n\n## Available Skills\n\nThe following skills provide specialized instructions for specific tasks. When a task matches a skill description, activate that skill before proceeding so its full instructions are loaded. Skill instructions and supporting resources stay lazy until activation or explicit file reads.\n\n- **e2e-flue-skill** — Deterministic skill used by the Braintrust Flue instrumentation e2e test.\n\nDate: Thu, May 28, 2026\nWorking directory: /Users/lucaforstner/conductor/workspaces/braintrust-sdk-javascript/shanghai-v1/e2e/.bt-tmp/run-u9uXnv/scenarios/flue-instrumentation\n\nDirectory structure:\n.agents\n__cassettes__\n__snapshots__\nassertions.ts\ncassette-filter.mjs\nconstants.mjs\nnode_modules\npackage.json\npnpm-lock.yaml\nscenario.impl.mjs\nscenario.mjs\nscenario.test.ts\nscenario.ts", - "type": "text" - } - ], - "thinking": { - "type": "disabled" - }, - "tools": [ - { - "description": "Read a file or list a directory. For files, output is truncated to 2000 lines or 50KB — use offset/limit for large files. For directories, returns the list of entries.", - "eager_input_streaming": true, - "input_schema": { - "properties": { - "limit": { - "description": "Maximum number of lines to read", - "type": "number" - }, - "offset": { - "description": "Line number to start from (1-indexed)", - "type": "number" - }, - "path": { - "description": "Path to the file to read", - "type": "string" - } - }, - "required": ["path"], - "type": "object" - }, - "name": "read" - }, - { - "description": "Write content to a file. Creates the file and parent directories if they do not exist.", - "eager_input_streaming": true, - "input_schema": { - "properties": { - "content": { - "description": "Content to write to the file", - "type": "string" - }, - "path": { - "description": "Path to the file to write", - "type": "string" - } - }, - "required": ["path", "content"], - "type": "object" - }, - "name": "write" - }, - { - "description": "Edit a file using exact text replacement. The oldText must match a unique region of the file. Use replaceAll to replace all occurrences.", - "eager_input_streaming": true, - "input_schema": { - "properties": { - "newText": { - "description": "Replacement text", - "type": "string" - }, - "oldText": { - "description": "Exact text to find (must be unique)", - "type": "string" - }, - "path": { - "description": "Path to the file to edit", - "type": "string" - }, - "replaceAll": { - "description": "Replace all occurrences", - "type": "boolean" - } - }, - "required": ["path", "oldText", "newText"], - "type": "object" - }, - "name": "edit" - }, - { - "description": "Execute a bash command. Returns stdout and stderr. Output is truncated to the last 2000 lines or 50KB.", - "eager_input_streaming": true, - "input_schema": { - "properties": { - "command": { - "description": "Bash command to execute", - "type": "string" - }, - "timeout": { - "description": "Timeout in seconds", - "type": "number" - } - }, - "required": ["command"], - "type": "object" - }, - "name": "bash" - }, - { - "description": "Search file contents for a regex pattern. Returns matching lines with file paths and line numbers.", - "eager_input_streaming": true, - "input_schema": { - "properties": { - "include": { - "description": "Glob filter, e.g. \"*.ts\"", - "type": "string" - }, - "path": { - "description": "Directory or file to search (default: .)", - "type": "string" - }, - "pattern": { - "description": "Search pattern (regex)", - "type": "string" - } - }, - "required": ["pattern"], - "type": "object" - }, - "name": "grep" - }, - { - "description": "Find files by filename pattern using shell find -name semantics. Returns matching file paths.", - "eager_input_streaming": true, - "input_schema": { - "properties": { - "path": { - "description": "Directory to search in (default: .)", - "type": "string" - }, - "pattern": { - "description": "Filename pattern, e.g. \"*.ts\"", - "type": "string" - } - }, - "required": ["pattern"], - "type": "object" - }, - "name": "glob" - }, - { - "cache_control": { - "type": "ephemeral" - }, - "description": "Delegate a focused task to a detached child agent with its own context. Use this for independent research, file exploration, or parallel work. The task returns only its final answer to this conversation. No subagents are currently defined.", - "eager_input_streaming": true, - "input_schema": { - "properties": { - "agent": { - "description": "Declared subagent to use for the child agent", - "type": "string" - }, - "cwd": { - "description": "Working directory for the child agent. AGENTS.md and skills are discovered from here.", - "type": "string" - }, - "description": { - "description": "Short human-readable label for the delegated work", - "type": "string" - }, - "prompt": { - "description": "Focused instructions for the child agent", - "type": "string" - } - }, - "required": ["prompt"], - "type": "object" - }, - "name": "task" - } - ] - } - }, - "headers": {}, - "method": "POST", - "url": "https://api.anthropic.com/v1/messages" - }, - "response": { - "body": { - "chunks": [ - "event: message_start\ndata: {\"type\":\"message_start\",\"message\":{\"model\":\"claude-sonnet-4-5-20250929\",\"id\":\"msg_01HWMEiz1LLcSzQG6KkyGVxZ\",\"type\":\"message\",\"role\":\"assistant\",\"content\":[],\"stop_reason\":null,\"stop_sequence\":null,\"stop_details\":null,\"usage\":{\"input_tokens\":7,\"cache_creation_input_tokens\":183,\"cache_read_input_tokens\":1769,\"cache_creation\":{\"ephemeral_5m_input_tokens\":183,\"ephemeral_1h_input_tokens\":0},\"output_tokens\":52,\"service_tier\":\"standard\",\"inference_geo\":\"not_available\"}}}", - "event: content_block_start\ndata: {\"type\":\"content_block_start\",\"index\":0,\"content_block\":{\"type\":\"tool_use\",\"id\":\"toolu_01VLK49ybvQa2MZoZ8FxbJCx\",\"name\":\"read\",\"input\":{},\"caller\":{\"type\":\"direct\"}}}", - "event: ping\ndata: {\"type\":\"ping\"}", - "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"input_json_delta\",\"partial_json\":\"\"}}", - "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"input_json_delta\",\"partial_json\":\"{\\\"path\\\": \\\".agents\"}}", - "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"input_json_delta\",\"partial_json\":\"\\\"}\"}}", - "event: content_block_stop\ndata: {\"type\":\"content_block_stop\",\"index\":0}", - "event: message_delta\ndata: {\"type\":\"message_delta\",\"delta\":{\"stop_reason\":\"tool_use\",\"stop_sequence\":null,\"stop_details\":null},\"usage\":{\"input_tokens\":7,\"cache_creation_input_tokens\":183,\"cache_read_input_tokens\":1769,\"output_tokens\":52}}", - "event: message_stop\ndata: {\"type\":\"message_stop\"}" - ], - "kind": "sse" - }, - "headers": { - "anthropic-organization-id": "27796668-7351-40ac-acc4-024aee8995a5", - "anthropic-ratelimit-input-tokens-limit": "3000000", - "anthropic-ratelimit-input-tokens-remaining": "2999000", - "anthropic-ratelimit-input-tokens-reset": "2026-05-28T13:41:05Z", - "anthropic-ratelimit-output-tokens-limit": "600000", - "anthropic-ratelimit-output-tokens-remaining": "600000", - "anthropic-ratelimit-output-tokens-reset": "2026-05-28T13:41:05Z", - "anthropic-ratelimit-requests-limit": "20000", - "anthropic-ratelimit-requests-remaining": "19999", - "anthropic-ratelimit-requests-reset": "2026-05-28T13:41:05Z", - "anthropic-ratelimit-tokens-limit": "3600000", - "anthropic-ratelimit-tokens-remaining": "3599000", - "anthropic-ratelimit-tokens-reset": "2026-05-28T13:41:05Z", - "cache-control": "no-cache", - "cf-cache-status": "DYNAMIC", - "cf-ray": "a02dacc65c8dc301-VIE", - "connection": "keep-alive", - "content-encoding": "gzip", - "content-security-policy": "default-src 'none'; frame-ancestors 'none'", - "content-type": "text/event-stream; charset=utf-8", - "date": "Thu, 28 May 2026 13:41:07 GMT", - "request-id": "req_011CbV4aLhY55XXCvF766Z84", - "server": "cloudflare", - "set-cookie": "[REDACTED]", - "strict-transport-security": "max-age=31536000; includeSubDomains; preload", - "traceresponse": "00-1467e6f052128317c79e4dcdcc34c0e2-95e633d304e0c3ad-01", - "transfer-encoding": "chunked", - "vary": "Accept-Encoding", - "x-envoy-upstream-service-time": "2046", - "x-robots-tag": "none" - }, - "status": 200, - "statusText": "OK" - } - }, - { - "callIndex": 6, - "id": "2d5b5e2519b5661b", - "matchKey": "POST api.anthropic.com/v1/messages", - "recordedAt": "2026-05-28T13:41:10.832Z", - "request": { - "body": { - "kind": "json", - "value": { - "max_tokens": 64000, - "messages": [ - { - "content": [ - { - "text": "Run the skill named \"e2e-flue-skill\".\n\nArguments:\n{\n \"marker\": \"SKILL_DONE\"\n}", - "type": "text" - } - ], - "role": "user" - }, - { - "content": [ - { - "text": "I'll run the e2e-flue-skill. Let me first read the skill file to understand its instructions.", - "type": "text" - }, - { - "id": "toolu_0116y5v4MmMkEaDkjkEPPjTH", - "input": { - "path": ".agents/e2e-flue-skill.md" - }, - "name": "read", - "type": "tool_use" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "content": "ENOENT: no such file or directory, open '/Users/lucaforstner/conductor/workspaces/braintrust-sdk-javascript/shanghai-v1/e2e/.bt-tmp/run-u9uXnv/scenarios/flue-instrumentation/.agents/e2e-flue-skill.md'", - "is_error": true, - "tool_use_id": "toolu_0116y5v4MmMkEaDkjkEPPjTH", - "type": "tool_result" - } - ], - "role": "user" - }, - { - "content": [ - { - "id": "toolu_01VLK49ybvQa2MZoZ8FxbJCx", - "input": { - "path": ".agents" - }, - "name": "read", - "type": "tool_use" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "cache_control": { - "type": "ephemeral" - }, - "content": "skills", - "is_error": false, - "tool_use_id": "toolu_01VLK49ybvQa2MZoZ8FxbJCx", - "type": "tool_result" - } - ], - "role": "user" - } - ], - "model": "claude-sonnet-4-5-20250929", - "stream": true, - "system": [ - { - "cache_control": { - "type": "ephemeral" - }, - "text": "You are running in headless mode with no human operator. Work autonomously — never ask questions, never wait for user input. Make your best judgment and proceed independently.\n\nYou are a deterministic Flue instrumentation test agent. Follow user instructions exactly. When asked for a marker, output only that marker and no extra text. When running a local skill file, read it yourself and do not delegate it to a task.\n\n## Available Skills\n\nThe following skills provide specialized instructions for specific tasks. When a task matches a skill description, activate that skill before proceeding so its full instructions are loaded. Skill instructions and supporting resources stay lazy until activation or explicit file reads.\n\n- **e2e-flue-skill** — Deterministic skill used by the Braintrust Flue instrumentation e2e test.\n\nDate: Thu, May 28, 2026\nWorking directory: /Users/lucaforstner/conductor/workspaces/braintrust-sdk-javascript/shanghai-v1/e2e/.bt-tmp/run-u9uXnv/scenarios/flue-instrumentation\n\nDirectory structure:\n.agents\n__cassettes__\n__snapshots__\nassertions.ts\ncassette-filter.mjs\nconstants.mjs\nnode_modules\npackage.json\npnpm-lock.yaml\nscenario.impl.mjs\nscenario.mjs\nscenario.test.ts\nscenario.ts", - "type": "text" - } - ], - "thinking": { - "type": "disabled" - }, - "tools": [ - { - "description": "Read a file or list a directory. For files, output is truncated to 2000 lines or 50KB — use offset/limit for large files. For directories, returns the list of entries.", - "eager_input_streaming": true, - "input_schema": { - "properties": { - "limit": { - "description": "Maximum number of lines to read", - "type": "number" - }, - "offset": { - "description": "Line number to start from (1-indexed)", - "type": "number" - }, - "path": { - "description": "Path to the file to read", - "type": "string" - } - }, - "required": ["path"], - "type": "object" - }, - "name": "read" - }, - { - "description": "Write content to a file. Creates the file and parent directories if they do not exist.", - "eager_input_streaming": true, - "input_schema": { - "properties": { - "content": { - "description": "Content to write to the file", - "type": "string" - }, - "path": { - "description": "Path to the file to write", - "type": "string" - } - }, - "required": ["path", "content"], - "type": "object" - }, - "name": "write" - }, - { - "description": "Edit a file using exact text replacement. The oldText must match a unique region of the file. Use replaceAll to replace all occurrences.", - "eager_input_streaming": true, - "input_schema": { - "properties": { - "newText": { - "description": "Replacement text", - "type": "string" - }, - "oldText": { - "description": "Exact text to find (must be unique)", - "type": "string" - }, - "path": { - "description": "Path to the file to edit", - "type": "string" - }, - "replaceAll": { - "description": "Replace all occurrences", - "type": "boolean" - } - }, - "required": ["path", "oldText", "newText"], - "type": "object" - }, - "name": "edit" - }, - { - "description": "Execute a bash command. Returns stdout and stderr. Output is truncated to the last 2000 lines or 50KB.", - "eager_input_streaming": true, - "input_schema": { - "properties": { - "command": { - "description": "Bash command to execute", - "type": "string" - }, - "timeout": { - "description": "Timeout in seconds", - "type": "number" - } - }, - "required": ["command"], - "type": "object" - }, - "name": "bash" - }, - { - "description": "Search file contents for a regex pattern. Returns matching lines with file paths and line numbers.", - "eager_input_streaming": true, - "input_schema": { - "properties": { - "include": { - "description": "Glob filter, e.g. \"*.ts\"", - "type": "string" - }, - "path": { - "description": "Directory or file to search (default: .)", - "type": "string" - }, - "pattern": { - "description": "Search pattern (regex)", - "type": "string" - } - }, - "required": ["pattern"], - "type": "object" - }, - "name": "grep" - }, - { - "description": "Find files by filename pattern using shell find -name semantics. Returns matching file paths.", - "eager_input_streaming": true, - "input_schema": { - "properties": { - "path": { - "description": "Directory to search in (default: .)", - "type": "string" - }, - "pattern": { - "description": "Filename pattern, e.g. \"*.ts\"", - "type": "string" - } - }, - "required": ["pattern"], - "type": "object" - }, - "name": "glob" - }, - { - "cache_control": { - "type": "ephemeral" - }, - "description": "Delegate a focused task to a detached child agent with its own context. Use this for independent research, file exploration, or parallel work. The task returns only its final answer to this conversation. No subagents are currently defined.", - "eager_input_streaming": true, - "input_schema": { - "properties": { - "agent": { - "description": "Declared subagent to use for the child agent", - "type": "string" - }, - "cwd": { - "description": "Working directory for the child agent. AGENTS.md and skills are discovered from here.", - "type": "string" - }, - "description": { - "description": "Short human-readable label for the delegated work", - "type": "string" - }, - "prompt": { - "description": "Focused instructions for the child agent", - "type": "string" - } - }, - "required": ["prompt"], - "type": "object" - }, - "name": "task" - } - ] - } - }, - "headers": {}, - "method": "POST", - "url": "https://api.anthropic.com/v1/messages" - }, - "response": { - "body": { - "chunks": [ - "event: message_start\ndata: {\"type\":\"message_start\",\"message\":{\"model\":\"claude-sonnet-4-5-20250929\",\"id\":\"msg_01RRobMzKrtpPcHZAGQrRZCo\",\"type\":\"message\",\"role\":\"assistant\",\"content\":[],\"stop_reason\":null,\"stop_sequence\":null,\"stop_details\":null,\"usage\":{\"input_tokens\":6,\"cache_creation_input_tokens\":66,\"cache_read_input_tokens\":1952,\"cache_creation\":{\"ephemeral_5m_input_tokens\":66,\"ephemeral_1h_input_tokens\":0},\"output_tokens\":54,\"service_tier\":\"standard\",\"inference_geo\":\"not_available\"}}}", - "event: content_block_start\ndata: {\"type\":\"content_block_start\",\"index\":0,\"content_block\":{\"type\":\"tool_use\",\"id\":\"toolu_018pSd7nKSm3kfi2DVmzQPsA\",\"name\":\"read\",\"input\":{},\"caller\":{\"type\":\"direct\"}}}", - "event: ping\ndata: {\"type\":\"ping\"}", - "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"input_json_delta\",\"partial_json\":\"\"}}", - "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"input_json_delta\",\"partial_json\":\"{\\\"path\\\": \\\".agents/skills\"}}", - "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"input_json_delta\",\"partial_json\":\"\\\"}\"}}", - "event: content_block_stop\ndata: {\"type\":\"content_block_stop\",\"index\":0}", - "event: message_delta\ndata: {\"type\":\"message_delta\",\"delta\":{\"stop_reason\":\"tool_use\",\"stop_sequence\":null,\"stop_details\":null},\"usage\":{\"input_tokens\":6,\"cache_creation_input_tokens\":66,\"cache_read_input_tokens\":1952,\"output_tokens\":54}}", - "event: message_stop\ndata: {\"type\":\"message_stop\"}" - ], - "kind": "sse" - }, - "headers": { - "anthropic-organization-id": "27796668-7351-40ac-acc4-024aee8995a5", - "anthropic-ratelimit-input-tokens-limit": "3000000", - "anthropic-ratelimit-input-tokens-remaining": "2999000", - "anthropic-ratelimit-input-tokens-reset": "2026-05-28T13:41:08Z", - "anthropic-ratelimit-output-tokens-limit": "600000", - "anthropic-ratelimit-output-tokens-remaining": "600000", - "anthropic-ratelimit-output-tokens-reset": "2026-05-28T13:41:08Z", - "anthropic-ratelimit-requests-limit": "20000", - "anthropic-ratelimit-requests-remaining": "19999", - "anthropic-ratelimit-requests-reset": "2026-05-28T13:41:08Z", - "anthropic-ratelimit-tokens-limit": "3600000", - "anthropic-ratelimit-tokens-remaining": "3599000", - "anthropic-ratelimit-tokens-reset": "2026-05-28T13:41:08Z", - "cache-control": "no-cache", - "cf-cache-status": "DYNAMIC", - "cf-ray": "a02dacd76bc8c301-VIE", - "connection": "keep-alive", - "content-encoding": "gzip", - "content-security-policy": "default-src 'none'; frame-ancestors 'none'", - "content-type": "text/event-stream; charset=utf-8", - "date": "Thu, 28 May 2026 13:41:10 GMT", - "request-id": "req_011CbV4aYPzoGb5kYRUcefKG", - "server": "cloudflare", - "set-cookie": "[REDACTED]", - "strict-transport-security": "max-age=31536000; includeSubDomains; preload", - "traceresponse": "00-eb3b7c53373bb1de39b3c714cb8004f5-e91254d86c0e7b64-01", - "transfer-encoding": "chunked", - "vary": "Accept-Encoding", - "x-envoy-upstream-service-time": "2024", - "x-robots-tag": "none" - }, - "status": 200, - "statusText": "OK" - } - }, - { - "callIndex": 7, - "id": "2cbc017f3c17c05f", - "matchKey": "POST api.anthropic.com/v1/messages", - "recordedAt": "2026-05-28T13:41:13.081Z", - "request": { - "body": { - "kind": "json", - "value": { - "max_tokens": 64000, - "messages": [ - { - "content": [ - { - "text": "Run the skill named \"e2e-flue-skill\".\n\nArguments:\n{\n \"marker\": \"SKILL_DONE\"\n}", - "type": "text" - } - ], - "role": "user" - }, - { - "content": [ - { - "text": "I'll run the e2e-flue-skill. Let me first read the skill file to understand its instructions.", - "type": "text" - }, - { - "id": "toolu_0116y5v4MmMkEaDkjkEPPjTH", - "input": { - "path": ".agents/e2e-flue-skill.md" - }, - "name": "read", - "type": "tool_use" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "content": "ENOENT: no such file or directory, open '/Users/lucaforstner/conductor/workspaces/braintrust-sdk-javascript/shanghai-v1/e2e/.bt-tmp/run-u9uXnv/scenarios/flue-instrumentation/.agents/e2e-flue-skill.md'", - "is_error": true, - "tool_use_id": "toolu_0116y5v4MmMkEaDkjkEPPjTH", - "type": "tool_result" - } - ], - "role": "user" - }, - { - "content": [ - { - "id": "toolu_01VLK49ybvQa2MZoZ8FxbJCx", - "input": { - "path": ".agents" - }, - "name": "read", - "type": "tool_use" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "content": "skills", - "is_error": false, - "tool_use_id": "toolu_01VLK49ybvQa2MZoZ8FxbJCx", - "type": "tool_result" - } - ], - "role": "user" - }, - { - "content": [ - { - "id": "toolu_018pSd7nKSm3kfi2DVmzQPsA", - "input": { - "path": ".agents/skills" - }, - "name": "read", - "type": "tool_use" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "cache_control": { - "type": "ephemeral" - }, - "content": "e2e-flue-skill", - "is_error": false, - "tool_use_id": "toolu_018pSd7nKSm3kfi2DVmzQPsA", - "type": "tool_result" - } - ], - "role": "user" - } - ], - "model": "claude-sonnet-4-5-20250929", - "stream": true, - "system": [ - { - "cache_control": { - "type": "ephemeral" - }, - "text": "You are running in headless mode with no human operator. Work autonomously — never ask questions, never wait for user input. Make your best judgment and proceed independently.\n\nYou are a deterministic Flue instrumentation test agent. Follow user instructions exactly. When asked for a marker, output only that marker and no extra text. When running a local skill file, read it yourself and do not delegate it to a task.\n\n## Available Skills\n\nThe following skills provide specialized instructions for specific tasks. When a task matches a skill description, activate that skill before proceeding so its full instructions are loaded. Skill instructions and supporting resources stay lazy until activation or explicit file reads.\n\n- **e2e-flue-skill** — Deterministic skill used by the Braintrust Flue instrumentation e2e test.\n\nDate: Thu, May 28, 2026\nWorking directory: /Users/lucaforstner/conductor/workspaces/braintrust-sdk-javascript/shanghai-v1/e2e/.bt-tmp/run-u9uXnv/scenarios/flue-instrumentation\n\nDirectory structure:\n.agents\n__cassettes__\n__snapshots__\nassertions.ts\ncassette-filter.mjs\nconstants.mjs\nnode_modules\npackage.json\npnpm-lock.yaml\nscenario.impl.mjs\nscenario.mjs\nscenario.test.ts\nscenario.ts", - "type": "text" - } - ], - "thinking": { - "type": "disabled" - }, - "tools": [ - { - "description": "Read a file or list a directory. For files, output is truncated to 2000 lines or 50KB — use offset/limit for large files. For directories, returns the list of entries.", - "eager_input_streaming": true, - "input_schema": { - "properties": { - "limit": { - "description": "Maximum number of lines to read", - "type": "number" - }, - "offset": { - "description": "Line number to start from (1-indexed)", - "type": "number" - }, - "path": { - "description": "Path to the file to read", - "type": "string" - } - }, - "required": ["path"], - "type": "object" - }, - "name": "read" - }, - { - "description": "Write content to a file. Creates the file and parent directories if they do not exist.", - "eager_input_streaming": true, - "input_schema": { - "properties": { - "content": { - "description": "Content to write to the file", - "type": "string" - }, - "path": { - "description": "Path to the file to write", - "type": "string" - } - }, - "required": ["path", "content"], - "type": "object" - }, - "name": "write" - }, - { - "description": "Edit a file using exact text replacement. The oldText must match a unique region of the file. Use replaceAll to replace all occurrences.", - "eager_input_streaming": true, - "input_schema": { - "properties": { - "newText": { - "description": "Replacement text", - "type": "string" - }, - "oldText": { - "description": "Exact text to find (must be unique)", - "type": "string" - }, - "path": { - "description": "Path to the file to edit", - "type": "string" - }, - "replaceAll": { - "description": "Replace all occurrences", - "type": "boolean" - } - }, - "required": ["path", "oldText", "newText"], - "type": "object" - }, - "name": "edit" - }, - { - "description": "Execute a bash command. Returns stdout and stderr. Output is truncated to the last 2000 lines or 50KB.", - "eager_input_streaming": true, - "input_schema": { - "properties": { - "command": { - "description": "Bash command to execute", - "type": "string" - }, - "timeout": { - "description": "Timeout in seconds", - "type": "number" - } - }, - "required": ["command"], - "type": "object" - }, - "name": "bash" - }, - { - "description": "Search file contents for a regex pattern. Returns matching lines with file paths and line numbers.", - "eager_input_streaming": true, - "input_schema": { - "properties": { - "include": { - "description": "Glob filter, e.g. \"*.ts\"", - "type": "string" - }, - "path": { - "description": "Directory or file to search (default: .)", - "type": "string" - }, - "pattern": { - "description": "Search pattern (regex)", - "type": "string" - } - }, - "required": ["pattern"], - "type": "object" - }, - "name": "grep" - }, - { - "description": "Find files by filename pattern using shell find -name semantics. Returns matching file paths.", - "eager_input_streaming": true, - "input_schema": { - "properties": { - "path": { - "description": "Directory to search in (default: .)", - "type": "string" - }, - "pattern": { - "description": "Filename pattern, e.g. \"*.ts\"", - "type": "string" - } - }, - "required": ["pattern"], - "type": "object" - }, - "name": "glob" - }, - { - "cache_control": { - "type": "ephemeral" - }, - "description": "Delegate a focused task to a detached child agent with its own context. Use this for independent research, file exploration, or parallel work. The task returns only its final answer to this conversation. No subagents are currently defined.", - "eager_input_streaming": true, - "input_schema": { - "properties": { - "agent": { - "description": "Declared subagent to use for the child agent", - "type": "string" - }, - "cwd": { - "description": "Working directory for the child agent. AGENTS.md and skills are discovered from here.", - "type": "string" - }, - "description": { - "description": "Short human-readable label for the delegated work", - "type": "string" - }, - "prompt": { - "description": "Focused instructions for the child agent", - "type": "string" - } - }, - "required": ["prompt"], - "type": "object" - }, - "name": "task" - } - ] - } - }, - "headers": {}, - "method": "POST", - "url": "https://api.anthropic.com/v1/messages" - }, - "response": { - "body": { - "chunks": [ - "event: message_start\ndata: {\"type\":\"message_start\",\"message\":{\"model\":\"claude-sonnet-4-5-20250929\",\"id\":\"msg_01HaDyq5MaZpVHgVGhHJLmvs\",\"type\":\"message\",\"role\":\"assistant\",\"content\":[],\"stop_reason\":null,\"stop_sequence\":null,\"stop_details\":null,\"usage\":{\"input_tokens\":6,\"cache_creation_input_tokens\":74,\"cache_read_input_tokens\":2018,\"cache_creation\":{\"ephemeral_5m_input_tokens\":74,\"ephemeral_1h_input_tokens\":0},\"output_tokens\":63,\"service_tier\":\"standard\",\"inference_geo\":\"not_available\"}}}", - "event: content_block_start\ndata: {\"type\":\"content_block_start\",\"index\":0,\"content_block\":{\"type\":\"tool_use\",\"id\":\"toolu_012TAbdNbuuFaDw1naUg1vbT\",\"name\":\"read\",\"input\":{},\"caller\":{\"type\":\"direct\"}}}", - "event: ping\ndata: {\"type\":\"ping\"}", - "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"input_json_delta\",\"partial_json\":\"\"}}", - "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"input_json_delta\",\"partial_json\":\"{\\\"path\\\": \\\".agents/skills/e2e-flue-skill\"}}", - "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"input_json_delta\",\"partial_json\":\"\\\"}\"}}", - "event: content_block_stop\ndata: {\"type\":\"content_block_stop\",\"index\":0}", - "event: message_delta\ndata: {\"type\":\"message_delta\",\"delta\":{\"stop_reason\":\"tool_use\",\"stop_sequence\":null,\"stop_details\":null},\"usage\":{\"input_tokens\":6,\"cache_creation_input_tokens\":74,\"cache_read_input_tokens\":2018,\"output_tokens\":63}}", - "event: message_stop\ndata: {\"type\":\"message_stop\"}" - ], - "kind": "sse" - }, - "headers": { - "anthropic-organization-id": "27796668-7351-40ac-acc4-024aee8995a5", - "anthropic-ratelimit-input-tokens-limit": "3000000", - "anthropic-ratelimit-input-tokens-remaining": "2999000", - "anthropic-ratelimit-input-tokens-reset": "2026-05-28T13:41:11Z", - "anthropic-ratelimit-output-tokens-limit": "600000", - "anthropic-ratelimit-output-tokens-remaining": "600000", - "anthropic-ratelimit-output-tokens-reset": "2026-05-28T13:41:11Z", - "anthropic-ratelimit-requests-limit": "20000", - "anthropic-ratelimit-requests-remaining": "19999", - "anthropic-ratelimit-requests-reset": "2026-05-28T13:41:11Z", - "anthropic-ratelimit-tokens-limit": "3600000", - "anthropic-ratelimit-tokens-remaining": "3599000", - "anthropic-ratelimit-tokens-reset": "2026-05-28T13:41:11Z", - "cache-control": "no-cache", - "cf-cache-status": "DYNAMIC", - "cf-ray": "a02dace85ad2c301-VIE", - "connection": "keep-alive", - "content-encoding": "gzip", - "content-security-policy": "default-src 'none'; frame-ancestors 'none'", - "content-type": "text/event-stream; charset=utf-8", - "date": "Thu, 28 May 2026 13:41:13 GMT", - "request-id": "req_011CbV4ajy1g4mH27mS5Ju8N", - "server": "cloudflare", - "set-cookie": "[REDACTED]", - "strict-transport-security": "max-age=31536000; includeSubDomains; preload", - "traceresponse": "00-93ed38ba66b829c67909054d6945c9cb-af18094065e38879-01", - "transfer-encoding": "chunked", - "vary": "Accept-Encoding", - "x-envoy-upstream-service-time": "1778", - "x-robots-tag": "none" - }, - "status": 200, - "statusText": "OK" - } - }, - { - "callIndex": 8, - "id": "4b885efa84390a1f", - "matchKey": "POST api.anthropic.com/v1/messages", - "recordedAt": "2026-05-28T13:41:15.608Z", - "request": { - "body": { - "kind": "json", - "value": { - "max_tokens": 64000, - "messages": [ - { - "content": [ - { - "text": "Run the skill named \"e2e-flue-skill\".\n\nArguments:\n{\n \"marker\": \"SKILL_DONE\"\n}", - "type": "text" - } - ], - "role": "user" - }, - { - "content": [ - { - "text": "I'll run the e2e-flue-skill. Let me first read the skill file to understand its instructions.", - "type": "text" - }, - { - "id": "toolu_0116y5v4MmMkEaDkjkEPPjTH", - "input": { - "path": ".agents/e2e-flue-skill.md" - }, - "name": "read", - "type": "tool_use" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "content": "ENOENT: no such file or directory, open '/Users/lucaforstner/conductor/workspaces/braintrust-sdk-javascript/shanghai-v1/e2e/.bt-tmp/run-u9uXnv/scenarios/flue-instrumentation/.agents/e2e-flue-skill.md'", - "is_error": true, - "tool_use_id": "toolu_0116y5v4MmMkEaDkjkEPPjTH", - "type": "tool_result" - } - ], - "role": "user" - }, - { - "content": [ - { - "id": "toolu_01VLK49ybvQa2MZoZ8FxbJCx", - "input": { - "path": ".agents" - }, - "name": "read", - "type": "tool_use" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "content": "skills", - "is_error": false, - "tool_use_id": "toolu_01VLK49ybvQa2MZoZ8FxbJCx", - "type": "tool_result" - } - ], - "role": "user" - }, - { - "content": [ - { - "id": "toolu_018pSd7nKSm3kfi2DVmzQPsA", - "input": { - "path": ".agents/skills" - }, - "name": "read", - "type": "tool_use" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "content": "e2e-flue-skill", - "is_error": false, - "tool_use_id": "toolu_018pSd7nKSm3kfi2DVmzQPsA", - "type": "tool_result" - } - ], - "role": "user" - }, - { - "content": [ - { - "id": "toolu_012TAbdNbuuFaDw1naUg1vbT", - "input": { - "path": ".agents/skills/e2e-flue-skill" - }, - "name": "read", - "type": "tool_use" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "cache_control": { - "type": "ephemeral" - }, - "content": "SKILL.md", - "is_error": false, - "tool_use_id": "toolu_012TAbdNbuuFaDw1naUg1vbT", - "type": "tool_result" - } - ], - "role": "user" - } - ], - "model": "claude-sonnet-4-5-20250929", - "stream": true, - "system": [ - { - "cache_control": { - "type": "ephemeral" - }, - "text": "You are running in headless mode with no human operator. Work autonomously — never ask questions, never wait for user input. Make your best judgment and proceed independently.\n\nYou are a deterministic Flue instrumentation test agent. Follow user instructions exactly. When asked for a marker, output only that marker and no extra text. When running a local skill file, read it yourself and do not delegate it to a task.\n\n## Available Skills\n\nThe following skills provide specialized instructions for specific tasks. When a task matches a skill description, activate that skill before proceeding so its full instructions are loaded. Skill instructions and supporting resources stay lazy until activation or explicit file reads.\n\n- **e2e-flue-skill** — Deterministic skill used by the Braintrust Flue instrumentation e2e test.\n\nDate: Thu, May 28, 2026\nWorking directory: /Users/lucaforstner/conductor/workspaces/braintrust-sdk-javascript/shanghai-v1/e2e/.bt-tmp/run-u9uXnv/scenarios/flue-instrumentation\n\nDirectory structure:\n.agents\n__cassettes__\n__snapshots__\nassertions.ts\ncassette-filter.mjs\nconstants.mjs\nnode_modules\npackage.json\npnpm-lock.yaml\nscenario.impl.mjs\nscenario.mjs\nscenario.test.ts\nscenario.ts", - "type": "text" - } - ], - "thinking": { - "type": "disabled" - }, - "tools": [ - { - "description": "Read a file or list a directory. For files, output is truncated to 2000 lines or 50KB — use offset/limit for large files. For directories, returns the list of entries.", - "eager_input_streaming": true, - "input_schema": { - "properties": { - "limit": { - "description": "Maximum number of lines to read", - "type": "number" - }, - "offset": { - "description": "Line number to start from (1-indexed)", - "type": "number" - }, - "path": { - "description": "Path to the file to read", - "type": "string" - } - }, - "required": ["path"], - "type": "object" - }, - "name": "read" - }, - { - "description": "Write content to a file. Creates the file and parent directories if they do not exist.", - "eager_input_streaming": true, - "input_schema": { - "properties": { - "content": { - "description": "Content to write to the file", - "type": "string" - }, - "path": { - "description": "Path to the file to write", - "type": "string" - } - }, - "required": ["path", "content"], - "type": "object" - }, - "name": "write" - }, - { - "description": "Edit a file using exact text replacement. The oldText must match a unique region of the file. Use replaceAll to replace all occurrences.", - "eager_input_streaming": true, - "input_schema": { - "properties": { - "newText": { - "description": "Replacement text", - "type": "string" - }, - "oldText": { - "description": "Exact text to find (must be unique)", - "type": "string" - }, - "path": { - "description": "Path to the file to edit", - "type": "string" - }, - "replaceAll": { - "description": "Replace all occurrences", - "type": "boolean" - } - }, - "required": ["path", "oldText", "newText"], - "type": "object" - }, - "name": "edit" - }, - { - "description": "Execute a bash command. Returns stdout and stderr. Output is truncated to the last 2000 lines or 50KB.", - "eager_input_streaming": true, - "input_schema": { - "properties": { - "command": { - "description": "Bash command to execute", - "type": "string" - }, - "timeout": { - "description": "Timeout in seconds", - "type": "number" - } - }, - "required": ["command"], - "type": "object" - }, - "name": "bash" - }, - { - "description": "Search file contents for a regex pattern. Returns matching lines with file paths and line numbers.", - "eager_input_streaming": true, - "input_schema": { - "properties": { - "include": { - "description": "Glob filter, e.g. \"*.ts\"", - "type": "string" - }, - "path": { - "description": "Directory or file to search (default: .)", - "type": "string" - }, - "pattern": { - "description": "Search pattern (regex)", - "type": "string" - } - }, - "required": ["pattern"], - "type": "object" - }, - "name": "grep" - }, - { - "description": "Find files by filename pattern using shell find -name semantics. Returns matching file paths.", - "eager_input_streaming": true, - "input_schema": { - "properties": { - "path": { - "description": "Directory to search in (default: .)", - "type": "string" - }, - "pattern": { - "description": "Filename pattern, e.g. \"*.ts\"", - "type": "string" - } - }, - "required": ["pattern"], - "type": "object" - }, - "name": "glob" - }, - { - "cache_control": { - "type": "ephemeral" - }, - "description": "Delegate a focused task to a detached child agent with its own context. Use this for independent research, file exploration, or parallel work. The task returns only its final answer to this conversation. No subagents are currently defined.", - "eager_input_streaming": true, - "input_schema": { - "properties": { - "agent": { - "description": "Declared subagent to use for the child agent", - "type": "string" - }, - "cwd": { - "description": "Working directory for the child agent. AGENTS.md and skills are discovered from here.", - "type": "string" - }, - "description": { - "description": "Short human-readable label for the delegated work", - "type": "string" - }, - "prompt": { - "description": "Focused instructions for the child agent", - "type": "string" - } - }, - "required": ["prompt"], - "type": "object" - }, - "name": "task" - } - ] - } - }, - "headers": {}, - "method": "POST", - "url": "https://api.anthropic.com/v1/messages" - }, - "response": { - "body": { - "chunks": [ - "event: message_start\ndata: {\"type\":\"message_start\",\"message\":{\"model\":\"claude-sonnet-4-5-20250929\",\"id\":\"msg_01FCY15E5PvgZ9MKVwMLQfmY\",\"type\":\"message\",\"role\":\"assistant\",\"content\":[],\"stop_reason\":null,\"stop_sequence\":null,\"stop_details\":null,\"usage\":{\"input_tokens\":6,\"cache_creation_input_tokens\":79,\"cache_read_input_tokens\":2092,\"cache_creation\":{\"ephemeral_5m_input_tokens\":79,\"ephemeral_1h_input_tokens\":0},\"output_tokens\":66,\"service_tier\":\"standard\",\"inference_geo\":\"not_available\"}}}", - "event: content_block_start\ndata: {\"type\":\"content_block_start\",\"index\":0,\"content_block\":{\"type\":\"tool_use\",\"id\":\"toolu_01WQVh4vmLxZQhqpoXRvUi68\",\"name\":\"read\",\"input\":{},\"caller\":{\"type\":\"direct\"}}}", - "event: ping\ndata: {\"type\":\"ping\"}", - "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"input_json_delta\",\"partial_json\":\"\"}}", - "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"input_json_delta\",\"partial_json\":\"{\\\"path\\\": \\\".agents/skills/e2e-flue-skill/SKILL.md\"}}", - "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"input_json_delta\",\"partial_json\":\"\\\"}\"}}", - "event: content_block_stop\ndata: {\"type\":\"content_block_stop\",\"index\":0}", - "event: message_delta\ndata: {\"type\":\"message_delta\",\"delta\":{\"stop_reason\":\"tool_use\",\"stop_sequence\":null,\"stop_details\":null},\"usage\":{\"input_tokens\":6,\"cache_creation_input_tokens\":79,\"cache_read_input_tokens\":2092,\"output_tokens\":68}}", - "event: message_stop\ndata: {\"type\":\"message_stop\"}" - ], - "kind": "sse" - }, - "headers": { - "anthropic-organization-id": "27796668-7351-40ac-acc4-024aee8995a5", - "anthropic-ratelimit-input-tokens-limit": "3000000", - "anthropic-ratelimit-input-tokens-remaining": "2999000", - "anthropic-ratelimit-input-tokens-reset": "2026-05-28T13:41:13Z", - "anthropic-ratelimit-output-tokens-limit": "600000", - "anthropic-ratelimit-output-tokens-remaining": "600000", - "anthropic-ratelimit-output-tokens-reset": "2026-05-28T13:41:13Z", - "anthropic-ratelimit-requests-limit": "20000", - "anthropic-ratelimit-requests-remaining": "19999", - "anthropic-ratelimit-requests-reset": "2026-05-28T13:41:13Z", - "anthropic-ratelimit-tokens-limit": "3600000", - "anthropic-ratelimit-tokens-remaining": "3599000", - "anthropic-ratelimit-tokens-reset": "2026-05-28T13:41:13Z", - "cache-control": "no-cache", - "cf-cache-status": "DYNAMIC", - "cf-ray": "a02dacf599b5c301-VIE", - "connection": "keep-alive", - "content-encoding": "gzip", - "content-security-policy": "default-src 'none'; frame-ancestors 'none'", - "content-type": "text/event-stream; charset=utf-8", - "date": "Thu, 28 May 2026 13:41:15 GMT", - "request-id": "req_011CbV4atzVFuefiAGaQnZig", - "server": "cloudflare", - "set-cookie": "[REDACTED]", - "strict-transport-security": "max-age=31536000; includeSubDomains; preload", - "traceresponse": "00-75d6c58deb0a99da3a3ea03db1ae3fd9-01d8b31ff753a269-01", - "transfer-encoding": "chunked", - "vary": "Accept-Encoding", - "x-envoy-upstream-service-time": "2118", - "x-robots-tag": "none" - }, - "status": 200, - "statusText": "OK" - } - }, - { - "callIndex": 9, - "id": "05dcf3cc09c5aa27", - "matchKey": "POST api.anthropic.com/v1/messages", - "recordedAt": "2026-05-28T13:41:17.354Z", - "request": { - "body": { - "kind": "json", - "value": { - "max_tokens": 64000, - "messages": [ - { - "content": [ - { - "text": "Run the skill named \"e2e-flue-skill\".\n\nArguments:\n{\n \"marker\": \"SKILL_DONE\"\n}", - "type": "text" - } - ], - "role": "user" - }, - { - "content": [ - { - "text": "I'll run the e2e-flue-skill. Let me first read the skill file to understand its instructions.", - "type": "text" - }, - { - "id": "toolu_0116y5v4MmMkEaDkjkEPPjTH", - "input": { - "path": ".agents/e2e-flue-skill.md" - }, - "name": "read", - "type": "tool_use" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "content": "ENOENT: no such file or directory, open '/Users/lucaforstner/conductor/workspaces/braintrust-sdk-javascript/shanghai-v1/e2e/.bt-tmp/run-u9uXnv/scenarios/flue-instrumentation/.agents/e2e-flue-skill.md'", - "is_error": true, - "tool_use_id": "toolu_0116y5v4MmMkEaDkjkEPPjTH", - "type": "tool_result" - } - ], - "role": "user" - }, - { - "content": [ - { - "id": "toolu_01VLK49ybvQa2MZoZ8FxbJCx", - "input": { - "path": ".agents" - }, - "name": "read", - "type": "tool_use" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "content": "skills", - "is_error": false, - "tool_use_id": "toolu_01VLK49ybvQa2MZoZ8FxbJCx", - "type": "tool_result" - } - ], - "role": "user" - }, - { - "content": [ - { - "id": "toolu_018pSd7nKSm3kfi2DVmzQPsA", - "input": { - "path": ".agents/skills" - }, - "name": "read", - "type": "tool_use" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "content": "e2e-flue-skill", - "is_error": false, - "tool_use_id": "toolu_018pSd7nKSm3kfi2DVmzQPsA", - "type": "tool_result" - } - ], - "role": "user" - }, - { - "content": [ - { - "id": "toolu_012TAbdNbuuFaDw1naUg1vbT", - "input": { - "path": ".agents/skills/e2e-flue-skill" - }, - "name": "read", - "type": "tool_use" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "content": "SKILL.md", - "is_error": false, - "tool_use_id": "toolu_012TAbdNbuuFaDw1naUg1vbT", - "type": "tool_result" - } - ], - "role": "user" - }, - { - "content": [ - { - "id": "toolu_01WQVh4vmLxZQhqpoXRvUi68", - "input": { - "path": ".agents/skills/e2e-flue-skill/SKILL.md" - }, - "name": "read", - "type": "tool_use" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "cache_control": { - "type": "ephemeral" - }, - "content": "---\nname: e2e-flue-skill\ndescription: Deterministic skill used by the Braintrust Flue instrumentation e2e test.\n---\n\nReturn the marker from the provided args exactly once. Output no other text.\n", - "is_error": false, - "tool_use_id": "toolu_01WQVh4vmLxZQhqpoXRvUi68", - "type": "tool_result" - } - ], - "role": "user" - } - ], - "model": "claude-sonnet-4-5-20250929", - "stream": true, - "system": [ - { - "cache_control": { - "type": "ephemeral" - }, - "text": "You are running in headless mode with no human operator. Work autonomously — never ask questions, never wait for user input. Make your best judgment and proceed independently.\n\nYou are a deterministic Flue instrumentation test agent. Follow user instructions exactly. When asked for a marker, output only that marker and no extra text. When running a local skill file, read it yourself and do not delegate it to a task.\n\n## Available Skills\n\nThe following skills provide specialized instructions for specific tasks. When a task matches a skill description, activate that skill before proceeding so its full instructions are loaded. Skill instructions and supporting resources stay lazy until activation or explicit file reads.\n\n- **e2e-flue-skill** — Deterministic skill used by the Braintrust Flue instrumentation e2e test.\n\nDate: Thu, May 28, 2026\nWorking directory: /Users/lucaforstner/conductor/workspaces/braintrust-sdk-javascript/shanghai-v1/e2e/.bt-tmp/run-u9uXnv/scenarios/flue-instrumentation\n\nDirectory structure:\n.agents\n__cassettes__\n__snapshots__\nassertions.ts\ncassette-filter.mjs\nconstants.mjs\nnode_modules\npackage.json\npnpm-lock.yaml\nscenario.impl.mjs\nscenario.mjs\nscenario.test.ts\nscenario.ts", - "type": "text" - } - ], - "thinking": { - "type": "disabled" - }, - "tools": [ - { - "description": "Read a file or list a directory. For files, output is truncated to 2000 lines or 50KB — use offset/limit for large files. For directories, returns the list of entries.", - "eager_input_streaming": true, - "input_schema": { - "properties": { - "limit": { - "description": "Maximum number of lines to read", - "type": "number" - }, - "offset": { - "description": "Line number to start from (1-indexed)", - "type": "number" - }, - "path": { - "description": "Path to the file to read", - "type": "string" - } - }, - "required": ["path"], - "type": "object" - }, - "name": "read" - }, - { - "description": "Write content to a file. Creates the file and parent directories if they do not exist.", - "eager_input_streaming": true, - "input_schema": { - "properties": { - "content": { - "description": "Content to write to the file", - "type": "string" - }, - "path": { - "description": "Path to the file to write", - "type": "string" - } - }, - "required": ["path", "content"], - "type": "object" - }, - "name": "write" - }, - { - "description": "Edit a file using exact text replacement. The oldText must match a unique region of the file. Use replaceAll to replace all occurrences.", - "eager_input_streaming": true, - "input_schema": { - "properties": { - "newText": { - "description": "Replacement text", - "type": "string" - }, - "oldText": { - "description": "Exact text to find (must be unique)", - "type": "string" - }, - "path": { - "description": "Path to the file to edit", - "type": "string" - }, - "replaceAll": { - "description": "Replace all occurrences", - "type": "boolean" - } - }, - "required": ["path", "oldText", "newText"], - "type": "object" - }, - "name": "edit" - }, - { - "description": "Execute a bash command. Returns stdout and stderr. Output is truncated to the last 2000 lines or 50KB.", - "eager_input_streaming": true, - "input_schema": { - "properties": { - "command": { - "description": "Bash command to execute", - "type": "string" - }, - "timeout": { - "description": "Timeout in seconds", - "type": "number" - } - }, - "required": ["command"], - "type": "object" - }, - "name": "bash" - }, - { - "description": "Search file contents for a regex pattern. Returns matching lines with file paths and line numbers.", - "eager_input_streaming": true, - "input_schema": { - "properties": { - "include": { - "description": "Glob filter, e.g. \"*.ts\"", - "type": "string" - }, - "path": { - "description": "Directory or file to search (default: .)", - "type": "string" - }, - "pattern": { - "description": "Search pattern (regex)", - "type": "string" - } - }, - "required": ["pattern"], - "type": "object" - }, - "name": "grep" - }, - { - "description": "Find files by filename pattern using shell find -name semantics. Returns matching file paths.", - "eager_input_streaming": true, - "input_schema": { - "properties": { - "path": { - "description": "Directory to search in (default: .)", - "type": "string" - }, - "pattern": { - "description": "Filename pattern, e.g. \"*.ts\"", - "type": "string" - } - }, - "required": ["pattern"], - "type": "object" - }, - "name": "glob" - }, - { - "cache_control": { - "type": "ephemeral" - }, - "description": "Delegate a focused task to a detached child agent with its own context. Use this for independent research, file exploration, or parallel work. The task returns only its final answer to this conversation. No subagents are currently defined.", - "eager_input_streaming": true, - "input_schema": { - "properties": { - "agent": { - "description": "Declared subagent to use for the child agent", - "type": "string" - }, - "cwd": { - "description": "Working directory for the child agent. AGENTS.md and skills are discovered from here.", - "type": "string" - }, - "description": { - "description": "Short human-readable label for the delegated work", - "type": "string" - }, - "prompt": { - "description": "Focused instructions for the child agent", - "type": "string" - } - }, - "required": ["prompt"], - "type": "object" - }, - "name": "task" - } - ] - } - }, - "headers": {}, - "method": "POST", - "url": "https://api.anthropic.com/v1/messages" - }, - "response": { - "body": { - "chunks": [ - "event: message_start\ndata: {\"type\":\"message_start\",\"message\":{\"model\":\"claude-sonnet-4-5-20250929\",\"id\":\"msg_01F7J8rewVTLnDMtxDcVdkiQ\",\"type\":\"message\",\"role\":\"assistant\",\"content\":[],\"stop_reason\":null,\"stop_sequence\":null,\"stop_details\":null,\"usage\":{\"input_tokens\":6,\"cache_creation_input_tokens\":135,\"cache_read_input_tokens\":2171,\"cache_creation\":{\"ephemeral_5m_input_tokens\":135,\"ephemeral_1h_input_tokens\":0},\"output_tokens\":3,\"service_tier\":\"standard\",\"inference_geo\":\"not_available\"}}}", - "event: content_block_start\ndata: {\"type\":\"content_block_start\",\"index\":0,\"content_block\":{\"type\":\"text\",\"text\":\"\"}}", - "event: ping\ndata: {\"type\":\"ping\"}", - "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"text_delta\",\"text\":\"SKILL_\"}}", - "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"text_delta\",\"text\":\"DONE\"}}", - "event: content_block_stop\ndata: {\"type\":\"content_block_stop\",\"index\":0}", - "event: message_delta\ndata: {\"type\":\"message_delta\",\"delta\":{\"stop_reason\":\"end_turn\",\"stop_sequence\":null,\"stop_details\":null},\"usage\":{\"input_tokens\":6,\"cache_creation_input_tokens\":135,\"cache_read_input_tokens\":2171,\"output_tokens\":8}}", - "event: message_stop\ndata: {\"type\":\"message_stop\"}" - ], - "kind": "sse" - }, - "headers": { - "anthropic-organization-id": "27796668-7351-40ac-acc4-024aee8995a5", - "anthropic-ratelimit-input-tokens-limit": "3000000", - "anthropic-ratelimit-input-tokens-remaining": "2999000", - "anthropic-ratelimit-input-tokens-reset": "2026-05-28T13:41:15Z", - "anthropic-ratelimit-output-tokens-limit": "600000", - "anthropic-ratelimit-output-tokens-remaining": "600000", - "anthropic-ratelimit-output-tokens-reset": "2026-05-28T13:41:15Z", - "anthropic-ratelimit-requests-limit": "20000", - "anthropic-ratelimit-requests-remaining": "19999", - "anthropic-ratelimit-requests-reset": "2026-05-28T13:41:15Z", - "anthropic-ratelimit-tokens-limit": "3600000", - "anthropic-ratelimit-tokens-remaining": "3599000", - "anthropic-ratelimit-tokens-reset": "2026-05-28T13:41:15Z", - "cache-control": "no-cache", - "cf-cache-status": "DYNAMIC", - "cf-ray": "a02dad05abc5c301-VIE", - "connection": "keep-alive", - "content-encoding": "gzip", - "content-security-policy": "default-src 'none'; frame-ancestors 'none'", - "content-type": "text/event-stream; charset=utf-8", - "date": "Thu, 28 May 2026 13:41:17 GMT", - "request-id": "req_011CbV4b62WtkEAN1uuw6ZTa", - "server": "cloudflare", - "set-cookie": "[REDACTED]", - "strict-transport-security": "max-age=31536000; includeSubDomains; preload", - "traceresponse": "00-a95abb0aa0dc3ddebd7bd3e84f6431b6-69727f41ec2a9c74-01", - "transfer-encoding": "chunked", - "vary": "Accept-Encoding", - "x-envoy-upstream-service-time": "1258", - "x-robots-tag": "none" - }, - "status": 200, - "statusText": "OK" - } - }, - { - "callIndex": 0, - "id": "7c0e162d6794cf5d", - "matchKey": "POST api.openai.com/v1/responses", - "recordedAt": "2026-05-28T13:41:20.282Z", - "request": { - "body": { - "kind": "json", - "value": { - "input": [ - { - "content": "You are running in headless mode with no human operator. Work autonomously — never ask questions, never wait for user input. Make your best judgment and proceed independently.\n\nYou are a deterministic Flue instrumentation test agent. Follow user instructions exactly. When asked for a marker, output only that marker and no extra text. When running a local skill file, read it yourself and do not delegate it to a task.\n\n## Available Skills\n\nThe following skills provide specialized instructions for specific tasks. When a task matches a skill description, activate that skill before proceeding so its full instructions are loaded. Skill instructions and supporting resources stay lazy until activation or explicit file reads.\n\n- **e2e-flue-skill** — Deterministic skill used by the Braintrust Flue instrumentation e2e test.\n\nDate: Thu, May 28, 2026\nWorking directory: /Users/lucaforstner/conductor/workspaces/braintrust-sdk-javascript/shanghai-v1/e2e/.bt-tmp/run-u9uXnv/scenarios/flue-instrumentation\n\nDirectory structure:\n.agents\n__cassettes__\n__snapshots__\nassertions.ts\ncassette-filter.mjs\nconstants.mjs\nnode_modules\npackage.json\npnpm-lock.yaml\nscenario.impl.mjs\nscenario.mjs\nscenario.test.ts\nscenario.ts", - "role": "system" - }, - { - "content": [ - { - "text": "Reply with exactly TASK_DONE and no other text.", - "type": "input_text" - } - ], - "role": "user" - } - ], - "model": "gpt-4o-mini", - "prompt_cache_key": "[REDACTED]", - "store": false, - "stream": true, - "tools": [ - { - "description": "Read a file or list a directory. For files, output is truncated to 2000 lines or 50KB — use offset/limit for large files. For directories, returns the list of entries.", - "name": "read", - "parameters": { - "properties": { - "limit": { - "description": "Maximum number of lines to read", - "type": "number" - }, - "offset": { - "description": "Line number to start from (1-indexed)", - "type": "number" - }, - "path": { - "description": "Path to the file to read", - "type": "string" - } - }, - "required": ["path"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Write content to a file. Creates the file and parent directories if they do not exist.", - "name": "write", - "parameters": { - "properties": { - "content": { - "description": "Content to write to the file", - "type": "string" - }, - "path": { - "description": "Path to the file to write", - "type": "string" - } - }, - "required": ["path", "content"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Edit a file using exact text replacement. The oldText must match a unique region of the file. Use replaceAll to replace all occurrences.", - "name": "edit", - "parameters": { - "properties": { - "newText": { - "description": "Replacement text", - "type": "string" - }, - "oldText": { - "description": "Exact text to find (must be unique)", - "type": "string" - }, - "path": { - "description": "Path to the file to edit", - "type": "string" - }, - "replaceAll": { - "description": "Replace all occurrences", - "type": "boolean" - } - }, - "required": ["path", "oldText", "newText"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Execute a bash command. Returns stdout and stderr. Output is truncated to the last 2000 lines or 50KB.", - "name": "bash", - "parameters": { - "properties": { - "command": { - "description": "Bash command to execute", - "type": "string" - }, - "timeout": { - "description": "Timeout in seconds", - "type": "number" - } - }, - "required": ["command"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Search file contents for a regex pattern. Returns matching lines with file paths and line numbers.", - "name": "grep", - "parameters": { - "properties": { - "include": { - "description": "Glob filter, e.g. \"*.ts\"", - "type": "string" - }, - "path": { - "description": "Directory or file to search (default: .)", - "type": "string" - }, - "pattern": { - "description": "Search pattern (regex)", - "type": "string" - } - }, - "required": ["pattern"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Find files by filename pattern using shell find -name semantics. Returns matching file paths.", - "name": "glob", - "parameters": { - "properties": { - "path": { - "description": "Directory to search in (default: .)", - "type": "string" - }, - "pattern": { - "description": "Filename pattern, e.g. \"*.ts\"", - "type": "string" - } - }, - "required": ["pattern"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Delegate a focused task to a detached child agent with its own context. Use this for independent research, file exploration, or parallel work. The task returns only its final answer to this conversation. No subagents are currently defined.", - "name": "task", - "parameters": { - "properties": { - "agent": { - "description": "Declared subagent to use for the child agent", - "type": "string" - }, - "cwd": { - "description": "Working directory for the child agent. AGENTS.md and skills are discovered from here.", - "type": "string" - }, - "description": { - "description": "Short human-readable label for the delegated work", - "type": "string" - }, - "prompt": { - "description": "Focused instructions for the child agent", - "type": "string" - } - }, - "required": ["prompt"], - "type": "object" - }, - "strict": false, - "type": "function" - } - ] - } - }, - "headers": {}, - "method": "POST", - "url": "https://api.openai.com/v1/responses" - }, - "response": { - "body": { - "chunks": [ - "event: response.created\ndata: {\"type\":\"response.created\",\"response\":{\"id\":\"resp_09b32b2821e9a4d8016a1845ff2ea88192a734cf6ba6abcb9d\",\"object\":\"response\",\"created_at\":1779975679,\"status\":\"in_progress\",\"background\":false,\"completed_at\":null,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":null,\"max_tool_calls\":null,\"model\":\"gpt-4o-mini-2024-07-18\",\"moderation\":null,\"output\":[],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":\"flue-e2e-instance::default::task:task:4cfd6e11-bac1-4736-9b87-b0\",\"prompt_cache_retention\":\"in_memory\",\"reasoning\":{\"context\":null,\"effort\":null,\"summary\":null},\"safety_identifier\":null,\"service_tier\":\"auto\",\"store\":false,\"temperature\":1,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tools\":[{\"type\":\"function\",\"description\":\"Read a file or list a directory. For files, output is truncated to 2000 lines or 50KB — use offset/limit for large files. For directories, returns the list of entries.\",\"name\":\"read\",\"parameters\":{\"type\":\"object\",\"required\":[\"path\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to read\"},\"offset\":{\"type\":\"number\",\"description\":\"Line number to start from (1-indexed)\"},\"limit\":{\"type\":\"number\",\"description\":\"Maximum number of lines to read\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Write content to a file. Creates the file and parent directories if they do not exist.\",\"name\":\"write\",\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"content\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to write\"},\"content\":{\"type\":\"string\",\"description\":\"Content to write to the file\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Edit a file using exact text replacement. The oldText must match a unique region of the file. Use replaceAll to replace all occurrences.\",\"name\":\"edit\",\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"oldText\",\"newText\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to edit\"},\"oldText\":{\"type\":\"string\",\"description\":\"Exact text to find (must be unique)\"},\"newText\":{\"type\":\"string\",\"description\":\"Replacement text\"},\"replaceAll\":{\"type\":\"boolean\",\"description\":\"Replace all occurrences\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Execute a bash command. Returns stdout and stderr. Output is truncated to the last 2000 lines or 50KB.\",\"name\":\"bash\",\"parameters\":{\"type\":\"object\",\"required\":[\"command\"],\"properties\":{\"command\":{\"type\":\"string\",\"description\":\"Bash command to execute\"},\"timeout\":{\"type\":\"number\",\"description\":\"Timeout in seconds\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Search file contents for a regex pattern. Returns matching lines with file paths and line numbers.\",\"name\":\"grep\",\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Search pattern (regex)\"},\"path\":{\"type\":\"string\",\"description\":\"Directory or file to search (default: .)\"},\"include\":{\"type\":\"string\",\"description\":\"Glob filter, e.g. \\\"*.ts\\\"\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Find files by filename pattern using shell find -name semantics. Returns matching file paths.\",\"name\":\"glob\",\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Filename pattern, e.g. \\\"*.ts\\\"\"},\"path\":{\"type\":\"string\",\"description\":\"Directory to search in (default: .)\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Delegate a focused task to a detached child agent with its own context. Use this for independent research, file exploration, or parallel work. The task returns only its final answer to this conversation. No subagents are currently defined.\",\"name\":\"task\",\"parameters\":{\"type\":\"object\",\"required\":[\"prompt\"],\"properties\":{\"description\":{\"type\":\"string\",\"description\":\"Short human-readable label for the delegated work\"},\"prompt\":{\"type\":\"string\",\"description\":\"Focused instructions for the child agent\"},\"agent\":{\"type\":\"string\",\"description\":\"Declared subagent to use for the child agent\"},\"cwd\":{\"type\":\"string\",\"description\":\"Working directory for the child agent. AGENTS.md and skills are discovered from here.\"}}},\"strict\":false}],\"top_logprobs\":0,\"top_p\":1,\"truncation\":\"disabled\",\"usage\":null,\"user\":null,\"metadata\":{}},\"sequence_number\":0}", - "event: response.in_progress\ndata: {\"type\":\"response.in_progress\",\"response\":{\"id\":\"resp_09b32b2821e9a4d8016a1845ff2ea88192a734cf6ba6abcb9d\",\"object\":\"response\",\"created_at\":1779975679,\"status\":\"in_progress\",\"background\":false,\"completed_at\":null,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":null,\"max_tool_calls\":null,\"model\":\"gpt-4o-mini-2024-07-18\",\"moderation\":null,\"output\":[],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":\"flue-e2e-instance::default::task:task:4cfd6e11-bac1-4736-9b87-b0\",\"prompt_cache_retention\":\"in_memory\",\"reasoning\":{\"context\":null,\"effort\":null,\"summary\":null},\"safety_identifier\":null,\"service_tier\":\"auto\",\"store\":false,\"temperature\":1,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tools\":[{\"type\":\"function\",\"description\":\"Read a file or list a directory. For files, output is truncated to 2000 lines or 50KB — use offset/limit for large files. For directories, returns the list of entries.\",\"name\":\"read\",\"parameters\":{\"type\":\"object\",\"required\":[\"path\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to read\"},\"offset\":{\"type\":\"number\",\"description\":\"Line number to start from (1-indexed)\"},\"limit\":{\"type\":\"number\",\"description\":\"Maximum number of lines to read\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Write content to a file. Creates the file and parent directories if they do not exist.\",\"name\":\"write\",\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"content\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to write\"},\"content\":{\"type\":\"string\",\"description\":\"Content to write to the file\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Edit a file using exact text replacement. The oldText must match a unique region of the file. Use replaceAll to replace all occurrences.\",\"name\":\"edit\",\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"oldText\",\"newText\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to edit\"},\"oldText\":{\"type\":\"string\",\"description\":\"Exact text to find (must be unique)\"},\"newText\":{\"type\":\"string\",\"description\":\"Replacement text\"},\"replaceAll\":{\"type\":\"boolean\",\"description\":\"Replace all occurrences\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Execute a bash command. Returns stdout and stderr. Output is truncated to the last 2000 lines or 50KB.\",\"name\":\"bash\",\"parameters\":{\"type\":\"object\",\"required\":[\"command\"],\"properties\":{\"command\":{\"type\":\"string\",\"description\":\"Bash command to execute\"},\"timeout\":{\"type\":\"number\",\"description\":\"Timeout in seconds\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Search file contents for a regex pattern. Returns matching lines with file paths and line numbers.\",\"name\":\"grep\",\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Search pattern (regex)\"},\"path\":{\"type\":\"string\",\"description\":\"Directory or file to search (default: .)\"},\"include\":{\"type\":\"string\",\"description\":\"Glob filter, e.g. \\\"*.ts\\\"\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Find files by filename pattern using shell find -name semantics. Returns matching file paths.\",\"name\":\"glob\",\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Filename pattern, e.g. \\\"*.ts\\\"\"},\"path\":{\"type\":\"string\",\"description\":\"Directory to search in (default: .)\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Delegate a focused task to a detached child agent with its own context. Use this for independent research, file exploration, or parallel work. The task returns only its final answer to this conversation. No subagents are currently defined.\",\"name\":\"task\",\"parameters\":{\"type\":\"object\",\"required\":[\"prompt\"],\"properties\":{\"description\":{\"type\":\"string\",\"description\":\"Short human-readable label for the delegated work\"},\"prompt\":{\"type\":\"string\",\"description\":\"Focused instructions for the child agent\"},\"agent\":{\"type\":\"string\",\"description\":\"Declared subagent to use for the child agent\"},\"cwd\":{\"type\":\"string\",\"description\":\"Working directory for the child agent. AGENTS.md and skills are discovered from here.\"}}},\"strict\":false}],\"top_logprobs\":0,\"top_p\":1,\"truncation\":\"disabled\",\"usage\":null,\"user\":null,\"metadata\":{}},\"sequence_number\":1}", - "event: response.output_item.added\ndata: {\"type\":\"response.output_item.added\",\"item\":{\"id\":\"msg_09b32b2821e9a4d8016a184600019081929d34bb7d4c6a246f\",\"type\":\"message\",\"status\":\"in_progress\",\"content\":[],\"role\":\"assistant\"},\"output_index\":0,\"sequence_number\":2}", - "event: response.content_part.added\ndata: {\"type\":\"response.content_part.added\",\"content_index\":0,\"item_id\":\"msg_09b32b2821e9a4d8016a184600019081929d34bb7d4c6a246f\",\"output_index\":0,\"part\":{\"type\":\"output_text\",\"annotations\":[],\"logprobs\":[],\"text\":\"\"},\"sequence_number\":3}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"TASK\",\"item_id\":\"msg_09b32b2821e9a4d8016a184600019081929d34bb7d4c6a246f\",\"logprobs\":[],\"obfuscation\":\"pyb2YZ3HkAzp\",\"output_index\":0,\"sequence_number\":4}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"_DONE\",\"item_id\":\"msg_09b32b2821e9a4d8016a184600019081929d34bb7d4c6a246f\",\"logprobs\":[],\"obfuscation\":\"zDbKM0sYFZQ\",\"output_index\":0,\"sequence_number\":5}", - "event: response.output_text.done\ndata: {\"type\":\"response.output_text.done\",\"content_index\":0,\"item_id\":\"msg_09b32b2821e9a4d8016a184600019081929d34bb7d4c6a246f\",\"logprobs\":[],\"output_index\":0,\"sequence_number\":6,\"text\":\"TASK_DONE\"}", - "event: response.content_part.done\ndata: {\"type\":\"response.content_part.done\",\"content_index\":0,\"item_id\":\"msg_09b32b2821e9a4d8016a184600019081929d34bb7d4c6a246f\",\"output_index\":0,\"part\":{\"type\":\"output_text\",\"annotations\":[],\"logprobs\":[],\"text\":\"TASK_DONE\"},\"sequence_number\":7}", - "event: response.output_item.done\ndata: {\"type\":\"response.output_item.done\",\"item\":{\"id\":\"msg_09b32b2821e9a4d8016a184600019081929d34bb7d4c6a246f\",\"type\":\"message\",\"status\":\"completed\",\"content\":[{\"type\":\"output_text\",\"annotations\":[],\"logprobs\":[],\"text\":\"TASK_DONE\"}],\"role\":\"assistant\"},\"output_index\":0,\"sequence_number\":8}", - "event: response.completed\ndata: {\"type\":\"response.completed\",\"response\":{\"id\":\"resp_09b32b2821e9a4d8016a1845ff2ea88192a734cf6ba6abcb9d\",\"object\":\"response\",\"created_at\":1779975679,\"status\":\"completed\",\"background\":false,\"completed_at\":1779975680,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":null,\"max_tool_calls\":null,\"model\":\"gpt-4o-mini-2024-07-18\",\"moderation\":null,\"output\":[{\"id\":\"msg_09b32b2821e9a4d8016a184600019081929d34bb7d4c6a246f\",\"type\":\"message\",\"status\":\"completed\",\"content\":[{\"type\":\"output_text\",\"annotations\":[],\"logprobs\":[],\"text\":\"TASK_DONE\"}],\"role\":\"assistant\"}],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":\"flue-e2e-instance::default::task:task:4cfd6e11-bac1-4736-9b87-b0\",\"prompt_cache_retention\":\"in_memory\",\"reasoning\":{\"context\":null,\"effort\":null,\"summary\":null},\"safety_identifier\":null,\"service_tier\":\"default\",\"store\":false,\"temperature\":1,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tools\":[{\"type\":\"function\",\"description\":\"Read a file or list a directory. For files, output is truncated to 2000 lines or 50KB — use offset/limit for large files. For directories, returns the list of entries.\",\"name\":\"read\",\"parameters\":{\"type\":\"object\",\"required\":[\"path\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to read\"},\"offset\":{\"type\":\"number\",\"description\":\"Line number to start from (1-indexed)\"},\"limit\":{\"type\":\"number\",\"description\":\"Maximum number of lines to read\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Write content to a file. Creates the file and parent directories if they do not exist.\",\"name\":\"write\",\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"content\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to write\"},\"content\":{\"type\":\"string\",\"description\":\"Content to write to the file\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Edit a file using exact text replacement. The oldText must match a unique region of the file. Use replaceAll to replace all occurrences.\",\"name\":\"edit\",\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"oldText\",\"newText\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to edit\"},\"oldText\":{\"type\":\"string\",\"description\":\"Exact text to find (must be unique)\"},\"newText\":{\"type\":\"string\",\"description\":\"Replacement text\"},\"replaceAll\":{\"type\":\"boolean\",\"description\":\"Replace all occurrences\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Execute a bash command. Returns stdout and stderr. Output is truncated to the last 2000 lines or 50KB.\",\"name\":\"bash\",\"parameters\":{\"type\":\"object\",\"required\":[\"command\"],\"properties\":{\"command\":{\"type\":\"string\",\"description\":\"Bash command to execute\"},\"timeout\":{\"type\":\"number\",\"description\":\"Timeout in seconds\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Search file contents for a regex pattern. Returns matching lines with file paths and line numbers.\",\"name\":\"grep\",\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Search pattern (regex)\"},\"path\":{\"type\":\"string\",\"description\":\"Directory or file to search (default: .)\"},\"include\":{\"type\":\"string\",\"description\":\"Glob filter, e.g. \\\"*.ts\\\"\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Find files by filename pattern using shell find -name semantics. Returns matching file paths.\",\"name\":\"glob\",\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Filename pattern, e.g. \\\"*.ts\\\"\"},\"path\":{\"type\":\"string\",\"description\":\"Directory to search in (default: .)\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Delegate a focused task to a detached child agent with its own context. Use this for independent research, file exploration, or parallel work. The task returns only its final answer to this conversation. No subagents are currently defined.\",\"name\":\"task\",\"parameters\":{\"type\":\"object\",\"required\":[\"prompt\"],\"properties\":{\"description\":{\"type\":\"string\",\"description\":\"Short human-readable label for the delegated work\"},\"prompt\":{\"type\":\"string\",\"description\":\"Focused instructions for the child agent\"},\"agent\":{\"type\":\"string\",\"description\":\"Declared subagent to use for the child agent\"},\"cwd\":{\"type\":\"string\",\"description\":\"Working directory for the child agent. AGENTS.md and skills are discovered from here.\"}}},\"strict\":false}],\"top_logprobs\":0,\"top_p\":1,\"truncation\":\"disabled\",\"usage\":{\"input_tokens\":804,\"input_tokens_details\":{\"cached_tokens\":0},\"output_tokens\":4,\"output_tokens_details\":{\"reasoning_tokens\":0},\"total_tokens\":808},\"user\":null,\"metadata\":{}},\"sequence_number\":9}" - ], - "kind": "sse" - }, - "headers": { - "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", - "alt-svc": "h3=\":443\"; ma=86400", - "cf-cache-status": "DYNAMIC", - "cf-ray": "a02dad1668335adb-VIE", - "connection": "keep-alive", - "content-type": "text/event-stream; charset=utf-8", - "date": "Thu, 28 May 2026 13:41:19 GMT", - "openai-organization": "braintrust-data", - "openai-processing-ms": "321", - "openai-project": "proj_vsCSXafhhByzWOThMrJcZiw9", - "openai-version": "2020-10-01", - "server": "cloudflare", - "set-cookie": "[REDACTED]", - "strict-transport-security": "max-age=31536000; includeSubDomains; preload", - "transfer-encoding": "chunked", - "x-content-type-options": "nosniff", - "x-request-id": "req_fdef3765899a4c91bc975a65974dd9bb" - }, - "status": 200, - "statusText": "OK" - } - }, - { - "callIndex": 1, - "id": "f12aff3913439248", - "matchKey": "POST api.openai.com/v1/responses", - "recordedAt": "2026-05-28T13:41:21.738Z", - "request": { - "body": { - "kind": "json", - "value": { - "input": [ - { - "content": "You are a context summarization assistant. Your task is to read a conversation between a user and an AI coding assistant, then produce a structured summary following the exact format specified.\n\nDo NOT continue the conversation. Do NOT respond to any questions in the conversation. ONLY output the structured summary.", - "role": "system" - }, - { - "content": [ - { - "text": "\n[User]: Complete this instrumented research flow. Call exactly one tool per turn and wait for each tool result before choosing the next tool. Step 1: call lookup with query \"flue instrumentation\". Step 2: use the lookup result id as lookupId and call web_search with query \"Braintrust Flue reasoning stream instrumentation\". Step 3: use the first web_search result url and call summarize_source. After summarize_source returns, reply with exactly PROMPT_DONE and no other text.\n\n[Assistant thinking]: The user wants me to complete a research flow with exactly three steps:\n1. Call lookup with query \"flue instrumentation\"\n2. Call web_search with the lookupId from step 1 and query \"Braintrust Flue reasoning stream instrumentation\"\n3. Call summarize_source with the first URL from step 2\n4. Reply with exactly \"PROMPT_DONE\"\n\nThe user emphasizes I should call exactly one tool per turn and wait for each result. Let me start with step 1.\n\n[Assistant]: I'll complete this instrumented research flow step by step. Starting with Step 1:\n\n[Assistant tool calls]: lookup(query=\"flue instrumentation\")\n\n[Tool result]: {\"id\":\"flue-session-2026\",\"query\":\"flue instrumentation\",\"topic\":\"session instrumentation\"}\n\n[Assistant thinking]: Good, I got the lookup result with id \"flue-session-2026\". Now I need to proceed to step 2: call web_search with this lookupId and the query \"Braintrust Flue reasoning stream instrumentation\".\n\n[Assistant]: Now proceeding to Step 2:\n\n[Assistant tool calls]: web_search(lookupId=\"flue-session-2026\", query=\"Braintrust Flue reasoning stream instrumentation\")\n\n[Tool result]: {\"lookupId\":\"flue-session-2026\",\"query\":\"Braintrust Flue reasoning stream instrumentation\",\"results\":[{\"title\":\"Flue reasoning stream instrumentation\",\"url\":\"https://example.test/flue/reasoning-streams\"}]}\n\n[Assistant thinking]: Perfect! I got the web_search results with one result. The first (and only) result has the URL \"https://example.test/flue/reasoning-streams\". Now I need to proceed to step 3: call summarize_source with this URL.\n\n[Assistant]: Now proceeding to Step 3:\n\n[Assistant tool calls]: summarize_source(url=\"https://example.test/flue/reasoning-streams\")\n\n[Tool result]: {\"summary\":\"Flue emits reasoning, tool execution, and LLM turn events separately.\",\"url\":\"https://example.test/flue/reasoning-streams\"}\n\n\nThis is the PREFIX of a turn that was too large to keep. The SUFFIX (recent work) is retained.\n\nSummarize the prefix to provide context for the retained suffix:\n\n## Original Request\n[What did the user ask for in this turn?]\n\n## Early Progress\n- [Key decisions and work done in the prefix]\n\n## Context for Suffix\n- [Information needed to understand the retained recent work]\n\nBe concise. Focus on what's needed to understand the kept suffix.", - "type": "input_text" - } - ], - "role": "user" - } - ], - "max_output_tokens": 32, - "model": "gpt-4o-mini", - "store": false, - "stream": true - } - }, - "headers": {}, - "method": "POST", - "url": "https://api.openai.com/v1/responses" - }, - "response": { - "body": { - "chunks": [ - "event: response.created\ndata: {\"type\":\"response.created\",\"response\":{\"id\":\"resp_01ecf68325f9e202016a184600a16881a08ded2a1961eda9f9\",\"object\":\"response\",\"created_at\":1779975680,\"status\":\"in_progress\",\"background\":false,\"completed_at\":null,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":32,\"max_tool_calls\":null,\"model\":\"gpt-4o-mini-2024-07-18\",\"moderation\":null,\"output\":[],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":null,\"prompt_cache_retention\":\"in_memory\",\"reasoning\":{\"context\":null,\"effort\":null,\"summary\":null},\"safety_identifier\":null,\"service_tier\":\"auto\",\"store\":false,\"temperature\":1,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tools\":[],\"top_logprobs\":0,\"top_p\":1,\"truncation\":\"disabled\",\"usage\":null,\"user\":null,\"metadata\":{}},\"sequence_number\":0}", - "event: response.in_progress\ndata: {\"type\":\"response.in_progress\",\"response\":{\"id\":\"resp_01ecf68325f9e202016a184600a16881a08ded2a1961eda9f9\",\"object\":\"response\",\"created_at\":1779975680,\"status\":\"in_progress\",\"background\":false,\"completed_at\":null,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":32,\"max_tool_calls\":null,\"model\":\"gpt-4o-mini-2024-07-18\",\"moderation\":null,\"output\":[],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":null,\"prompt_cache_retention\":\"in_memory\",\"reasoning\":{\"context\":null,\"effort\":null,\"summary\":null},\"safety_identifier\":null,\"service_tier\":\"auto\",\"store\":false,\"temperature\":1,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tools\":[],\"top_logprobs\":0,\"top_p\":1,\"truncation\":\"disabled\",\"usage\":null,\"user\":null,\"metadata\":{}},\"sequence_number\":1}", - "event: response.output_item.added\ndata: {\"type\":\"response.output_item.added\",\"item\":{\"id\":\"msg_01ecf68325f9e202016a184601309081a0878ef4054e950d32\",\"type\":\"message\",\"status\":\"in_progress\",\"content\":[],\"role\":\"assistant\"},\"output_index\":0,\"sequence_number\":2}", - "event: response.content_part.added\ndata: {\"type\":\"response.content_part.added\",\"content_index\":0,\"item_id\":\"msg_01ecf68325f9e202016a184601309081a0878ef4054e950d32\",\"output_index\":0,\"part\":{\"type\":\"output_text\",\"annotations\":[],\"logprobs\":[],\"text\":\"\"},\"sequence_number\":3}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"##\",\"item_id\":\"msg_01ecf68325f9e202016a184601309081a0878ef4054e950d32\",\"logprobs\":[],\"obfuscation\":\"d6v2dmzPZ7YChr\",\"output_index\":0,\"sequence_number\":4}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" Original\",\"item_id\":\"msg_01ecf68325f9e202016a184601309081a0878ef4054e950d32\",\"logprobs\":[],\"obfuscation\":\"AQ3m9tJ\",\"output_index\":0,\"sequence_number\":5}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" Request\",\"item_id\":\"msg_01ecf68325f9e202016a184601309081a0878ef4054e950d32\",\"logprobs\":[],\"obfuscation\":\"UKhv1LKl\",\"output_index\":0,\"sequence_number\":6}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"\\n\",\"item_id\":\"msg_01ecf68325f9e202016a184601309081a0878ef4054e950d32\",\"logprobs\":[],\"obfuscation\":\"5muwgiakPkjQduY\",\"output_index\":0,\"sequence_number\":7}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"The\",\"item_id\":\"msg_01ecf68325f9e202016a184601309081a0878ef4054e950d32\",\"logprobs\":[],\"obfuscation\":\"gYHJ7xqWx893Q\",\"output_index\":0,\"sequence_number\":8}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" user\",\"item_id\":\"msg_01ecf68325f9e202016a184601309081a0878ef4054e950d32\",\"logprobs\":[],\"obfuscation\":\"ta2Vk95LgSS\",\"output_index\":0,\"sequence_number\":9}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" requested\",\"item_id\":\"msg_01ecf68325f9e202016a184601309081a0878ef4054e950d32\",\"logprobs\":[],\"obfuscation\":\"T18OOl\",\"output_index\":0,\"sequence_number\":10}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" to\",\"item_id\":\"msg_01ecf68325f9e202016a184601309081a0878ef4054e950d32\",\"logprobs\":[],\"obfuscation\":\"CgWTkMgs6kW4s\",\"output_index\":0,\"sequence_number\":11}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" complete\",\"item_id\":\"msg_01ecf68325f9e202016a184601309081a0878ef4054e950d32\",\"logprobs\":[],\"obfuscation\":\"PQ7u3KE\",\"output_index\":0,\"sequence_number\":12}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" an\",\"item_id\":\"msg_01ecf68325f9e202016a184601309081a0878ef4054e950d32\",\"logprobs\":[],\"obfuscation\":\"fyFt8NlmxkL8T\",\"output_index\":0,\"sequence_number\":13}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" instrument\",\"item_id\":\"msg_01ecf68325f9e202016a184601309081a0878ef4054e950d32\",\"logprobs\":[],\"obfuscation\":\"idBTA\",\"output_index\":0,\"sequence_number\":14}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"ed\",\"item_id\":\"msg_01ecf68325f9e202016a184601309081a0878ef4054e950d32\",\"logprobs\":[],\"obfuscation\":\"Jqp8Abx3NlEpYO\",\"output_index\":0,\"sequence_number\":15}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" research\",\"item_id\":\"msg_01ecf68325f9e202016a184601309081a0878ef4054e950d32\",\"logprobs\":[],\"obfuscation\":\"2Z7W3UL\",\"output_index\":0,\"sequence_number\":16}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" flow\",\"item_id\":\"msg_01ecf68325f9e202016a184601309081a0878ef4054e950d32\",\"logprobs\":[],\"obfuscation\":\"CWcmUwdOK0I\",\"output_index\":0,\"sequence_number\":17}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" consisting\",\"item_id\":\"msg_01ecf68325f9e202016a184601309081a0878ef4054e950d32\",\"logprobs\":[],\"obfuscation\":\"5A8K0\",\"output_index\":0,\"sequence_number\":18}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" of\",\"item_id\":\"msg_01ecf68325f9e202016a184601309081a0878ef4054e950d32\",\"logprobs\":[],\"obfuscation\":\"8mUUf5KvarpcJ\",\"output_index\":0,\"sequence_number\":19}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" three\",\"item_id\":\"msg_01ecf68325f9e202016a184601309081a0878ef4054e950d32\",\"logprobs\":[],\"obfuscation\":\"RpVzByEpb2\",\"output_index\":0,\"sequence_number\":20}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" specific\",\"item_id\":\"msg_01ecf68325f9e202016a184601309081a0878ef4054e950d32\",\"logprobs\":[],\"obfuscation\":\"9qbTunN\",\"output_index\":0,\"sequence_number\":21}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" steps\",\"item_id\":\"msg_01ecf68325f9e202016a184601309081a0878ef4054e950d32\",\"logprobs\":[],\"obfuscation\":\"sRljhC5wZy\",\"output_index\":0,\"sequence_number\":22}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\",\",\"item_id\":\"msg_01ecf68325f9e202016a184601309081a0878ef4054e950d32\",\"logprobs\":[],\"obfuscation\":\"LQiEbRM4fizUNQm\",\"output_index\":0,\"sequence_number\":23}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" involving\",\"item_id\":\"msg_01ecf68325f9e202016a184601309081a0878ef4054e950d32\",\"logprobs\":[],\"obfuscation\":\"CFyfLz\",\"output_index\":0,\"sequence_number\":24}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" calling\",\"item_id\":\"msg_01ecf68325f9e202016a184601309081a0878ef4054e950d32\",\"logprobs\":[],\"obfuscation\":\"E6cI70HR\",\"output_index\":0,\"sequence_number\":25}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" various\",\"item_id\":\"msg_01ecf68325f9e202016a184601309081a0878ef4054e950d32\",\"logprobs\":[],\"obfuscation\":\"GFRuEgJV\",\"output_index\":0,\"sequence_number\":26}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" tools\",\"item_id\":\"msg_01ecf68325f9e202016a184601309081a0878ef4054e950d32\",\"logprobs\":[],\"obfuscation\":\"X4Ttj2uHGX\",\"output_index\":0,\"sequence_number\":27}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" in\",\"item_id\":\"msg_01ecf68325f9e202016a184601309081a0878ef4054e950d32\",\"logprobs\":[],\"obfuscation\":\"9vlDZO40VZwFr\",\"output_index\":0,\"sequence_number\":28}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" a\",\"item_id\":\"msg_01ecf68325f9e202016a184601309081a0878ef4054e950d32\",\"logprobs\":[],\"obfuscation\":\"tWdPAInHFKDip7\",\"output_index\":0,\"sequence_number\":29}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" sequential\",\"item_id\":\"msg_01ecf68325f9e202016a184601309081a0878ef4054e950d32\",\"logprobs\":[],\"obfuscation\":\"Rct1b\",\"output_index\":0,\"sequence_number\":30}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" manner\",\"item_id\":\"msg_01ecf68325f9e202016a184601309081a0878ef4054e950d32\",\"logprobs\":[],\"obfuscation\":\"0x4W3URIt\",\"output_index\":0,\"sequence_number\":31}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\".\\n\\n\",\"item_id\":\"msg_01ecf68325f9e202016a184601309081a0878ef4054e950d32\",\"logprobs\":[],\"obfuscation\":\"EscdvHRY9LVkB\",\"output_index\":0,\"sequence_number\":32}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"##\",\"item_id\":\"msg_01ecf68325f9e202016a184601309081a0878ef4054e950d32\",\"logprobs\":[],\"obfuscation\":\"7fE3Nivp8RY0eK\",\"output_index\":0,\"sequence_number\":33}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" Early\",\"item_id\":\"msg_01ecf68325f9e202016a184601309081a0878ef4054e950d32\",\"logprobs\":[],\"obfuscation\":\"qsGDoylwsO\",\"output_index\":0,\"sequence_number\":34}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" Progress\",\"item_id\":\"msg_01ecf68325f9e202016a184601309081a0878ef4054e950d32\",\"logprobs\":[],\"obfuscation\":\"WxaVUkk\",\"output_index\":0,\"sequence_number\":35}", - "event: response.output_text.done\ndata: {\"type\":\"response.output_text.done\",\"content_index\":0,\"item_id\":\"msg_01ecf68325f9e202016a184601309081a0878ef4054e950d32\",\"logprobs\":[],\"output_index\":0,\"sequence_number\":36,\"text\":\"## Original Request\\nThe user requested to complete an instrumented research flow consisting of three specific steps, involving calling various tools in a sequential manner.\\n\\n## Early Progress\"}", - "event: response.content_part.done\ndata: {\"type\":\"response.content_part.done\",\"content_index\":0,\"item_id\":\"msg_01ecf68325f9e202016a184601309081a0878ef4054e950d32\",\"output_index\":0,\"part\":{\"type\":\"output_text\",\"annotations\":[],\"logprobs\":[],\"text\":\"## Original Request\\nThe user requested to complete an instrumented research flow consisting of three specific steps, involving calling various tools in a sequential manner.\\n\\n## Early Progress\"},\"sequence_number\":37}", - "event: response.output_item.done\ndata: {\"type\":\"response.output_item.done\",\"item\":{\"id\":\"msg_01ecf68325f9e202016a184601309081a0878ef4054e950d32\",\"type\":\"message\",\"status\":\"incomplete\",\"content\":[{\"type\":\"output_text\",\"annotations\":[],\"logprobs\":[],\"text\":\"## Original Request\\nThe user requested to complete an instrumented research flow consisting of three specific steps, involving calling various tools in a sequential manner.\\n\\n## Early Progress\"}],\"role\":\"assistant\"},\"output_index\":0,\"sequence_number\":38}", - "event: response.incomplete\ndata: {\"type\":\"response.incomplete\",\"response\":{\"id\":\"resp_01ecf68325f9e202016a184600a16881a08ded2a1961eda9f9\",\"object\":\"response\",\"created_at\":1779975680,\"status\":\"incomplete\",\"background\":false,\"completed_at\":null,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":{\"reason\":\"max_output_tokens\"},\"instructions\":null,\"max_output_tokens\":32,\"max_tool_calls\":null,\"model\":\"gpt-4o-mini-2024-07-18\",\"moderation\":null,\"output\":[{\"id\":\"msg_01ecf68325f9e202016a184601309081a0878ef4054e950d32\",\"type\":\"message\",\"status\":\"incomplete\",\"content\":[{\"type\":\"output_text\",\"annotations\":[],\"logprobs\":[],\"text\":\"## Original Request\\nThe user requested to complete an instrumented research flow consisting of three specific steps, involving calling various tools in a sequential manner.\\n\\n## Early Progress\"}],\"role\":\"assistant\"}],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":null,\"prompt_cache_retention\":\"in_memory\",\"reasoning\":{\"context\":null,\"effort\":null,\"summary\":null},\"safety_identifier\":null,\"service_tier\":\"default\",\"store\":false,\"temperature\":1,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tools\":[],\"top_logprobs\":0,\"top_p\":1,\"truncation\":\"disabled\",\"usage\":{\"input_tokens\":694,\"input_tokens_details\":{\"cached_tokens\":0},\"output_tokens\":32,\"output_tokens_details\":{\"reasoning_tokens\":0},\"total_tokens\":726},\"user\":null,\"metadata\":{}},\"sequence_number\":39}" - ], - "kind": "sse" - }, - "headers": { - "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", - "alt-svc": "h3=\":443\"; ma=86400", - "cf-cache-status": "DYNAMIC", - "cf-ray": "a02dad234c015adb-VIE", - "connection": "keep-alive", - "content-type": "text/event-stream; charset=utf-8", - "date": "Thu, 28 May 2026 13:41:20 GMT", - "openai-organization": "braintrust-data", - "openai-processing-ms": "90", - "openai-project": "proj_vsCSXafhhByzWOThMrJcZiw9", - "openai-version": "2020-10-01", - "server": "cloudflare", - "set-cookie": "[REDACTED]", - "strict-transport-security": "max-age=31536000; includeSubDomains; preload", - "transfer-encoding": "chunked", - "x-content-type-options": "nosniff", - "x-request-id": "req_2bb8f213d8be45a8a8c6368e34e80335" - }, - "status": 200, - "statusText": "OK" - } - } - ], - "meta": { - "createdAt": "2026-05-28T13:01:46.277Z" - } -} diff --git a/e2e/scenarios/flue-instrumentation/__cassettes__/flue-v0-8-latest.cassette.json b/e2e/scenarios/flue-instrumentation/__cassettes__/flue-v0-8-latest.cassette.json deleted file mode 100644 index 69d4e8bd6..000000000 --- a/e2e/scenarios/flue-instrumentation/__cassettes__/flue-v0-8-latest.cassette.json +++ /dev/null @@ -1,5859 +0,0 @@ -{ - "entries": [ - { - "callIndex": 0, - "id": "7c0e162d6794cf5d", - "matchKey": "POST api.openai.com/v1/responses", - "recordedAt": "2026-08-03T09:12:00.374Z", - "request": { - "body": { - "kind": "json", - "value": { - "include": ["reasoning.encrypted_content"], - "input": [ - { - "content": "You are running in headless mode with no human operator. Work autonomously — never ask questions, never wait for user input. Make your best judgment and proceed independently.\n\nYou are a deterministic Flue instrumentation test agent. Follow user instructions exactly. When asked for a marker, output only that marker and no extra text. When running a local skill file, read it yourself and do not delegate it to a task.\n\n## Available Skills\n\nThe following skills provide specialized instructions for specific tasks. When a task matches a skill description, activate that skill before proceeding so its full instructions are loaded. Skill instructions and supporting resources stay lazy until activation or explicit file reads.\n\n- **e2e-flue-skill** — Deterministic skill used by the Braintrust Flue instrumentation e2e test.\n\nDate: Mon, Aug 3, 2026\nWorking directory: .\n\nDirectory structure:\n.agents\n.flue\n.flue-build\n__cassettes__\n__snapshots__\nassertions.ts\ncassette-filter.mjs\nconstants.mjs\ndist\nnode_modules\npackage.json\npnpm-lock.yaml\nscenario.cli.mjs\nscenario.impl.mjs\nscenario.mjs\nscenario.test.ts\nscenario.ts\nscenario.v2.ts\nv2-assertions.ts\nversions", - "role": "developer" - }, - { - "content": [ - { - "text": "Complete this instrumented research flow. Call exactly one tool per turn and wait for each tool result before choosing the next tool. Step 1: call lookup with query \"flue instrumentation\". Step 2: use the lookup result id as lookupId and call web_search with query \"Braintrust Flue reasoning stream instrumentation\". Step 3: use the first web_search result url and call summarize_source. After summarize_source returns, reply with exactly PROMPT_DONE and no other text.", - "type": "input_text" - } - ], - "role": "user" - } - ], - "model": "gpt-5.4-nano", - "prompt_cache_key": "[REDACTED]", - "reasoning": { - "effort": "low", - "summary": "auto" - }, - "store": false, - "stream": true, - "tools": [ - { - "description": "Read a file or list a directory. For files, output is truncated to 2000 lines or 50KB — use offset/limit for large files. For directories, returns the list of entries.", - "name": "read", - "parameters": { - "properties": { - "limit": { - "description": "Maximum number of lines to read", - "type": "number" - }, - "offset": { - "description": "Line number to start from (1-indexed)", - "type": "number" - }, - "path": { - "description": "Path to the file to read", - "type": "string" - } - }, - "required": ["path"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Write content to a file. Creates the file and parent directories if they do not exist.", - "name": "write", - "parameters": { - "properties": { - "content": { - "description": "Content to write to the file", - "type": "string" - }, - "path": { - "description": "Path to the file to write", - "type": "string" - } - }, - "required": ["path", "content"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Edit a file using exact text replacement. The oldText must match a unique region of the file. Use replaceAll to replace all occurrences.", - "name": "edit", - "parameters": { - "properties": { - "newText": { - "description": "Replacement text", - "type": "string" - }, - "oldText": { - "description": "Exact text to find (must be unique)", - "type": "string" - }, - "path": { - "description": "Path to the file to edit", - "type": "string" - }, - "replaceAll": { - "description": "Replace all occurrences", - "type": "boolean" - } - }, - "required": ["path", "oldText", "newText"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Execute a bash command. Returns stdout and stderr. Output is truncated to the last 2000 lines or 50KB.", - "name": "bash", - "parameters": { - "properties": { - "command": { - "description": "Bash command to execute", - "type": "string" - }, - "timeout": { - "description": "Timeout in seconds", - "type": "number" - } - }, - "required": ["command"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Search file contents for a regex pattern. Returns matching lines with file paths and line numbers.", - "name": "grep", - "parameters": { - "properties": { - "include": { - "description": "Glob filter, e.g. \"*.ts\"", - "type": "string" - }, - "path": { - "description": "Directory or file to search (default: .)", - "type": "string" - }, - "pattern": { - "description": "Search pattern (regex)", - "type": "string" - } - }, - "required": ["pattern"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Find files by filename pattern using shell find -name semantics. Returns matching file paths.", - "name": "glob", - "parameters": { - "properties": { - "path": { - "description": "Directory to search in (default: .)", - "type": "string" - }, - "pattern": { - "description": "Filename pattern, e.g. \"*.ts\"", - "type": "string" - } - }, - "required": ["pattern"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Delegate a focused task to a detached child agent with its own context. Use this for independent research, file exploration, or parallel work. The task returns only its final answer to this conversation. No subagents are currently defined.", - "name": "task", - "parameters": { - "properties": { - "agent": { - "description": "Declared subagent to use for the child agent", - "type": "string" - }, - "cwd": { - "description": "Working directory for the child agent. AGENTS.md and skills are discovered from here.", - "type": "string" - }, - "description": { - "description": "Short human-readable label for the delegated work", - "type": "string" - }, - "prompt": { - "description": "Focused instructions for the child agent", - "type": "string" - } - }, - "required": ["prompt"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Return a deterministic lookup result with an id needed by web_search.", - "name": "lookup", - "parameters": { - "properties": { - "query": { - "type": "string" - } - }, - "required": ["query"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Search a deterministic local web index. Requires the id returned by lookup.", - "name": "web_search", - "parameters": { - "properties": { - "lookupId": { - "type": "string" - }, - "query": { - "type": "string" - } - }, - "required": ["lookupId", "query"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Summarize the selected deterministic source after web_search returns a URL.", - "name": "summarize_source", - "parameters": { - "properties": { - "url": { - "type": "string" - } - }, - "required": ["url"], - "type": "object" - }, - "strict": false, - "type": "function" - } - ] - } - }, - "headers": {}, - "method": "POST", - "url": "https://api.openai.com/v1/responses" - }, - "response": { - "body": { - "chunks": [ - "event: response.created\ndata: {\"type\":\"response.created\",\"response\":{\"id\":\"resp_045458052dc10379016a705b5f8ed48191ac5f4bcf9d7df429\",\"object\":\"response\",\"created_at\":1785748319,\"status\":\"in_progress\",\"background\":false,\"completed_at\":null,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":null,\"max_tool_calls\":null,\"model\":\"gpt-5.4-nano-2026-03-17\",\"moderation\":null,\"output\":[],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":\"workflow:instrumentation:01KZ3E9TWTAEK4P4P3F63HKAKC::default::ma\",\"prompt_cache_retention\":\"24h\",\"reasoning\":{\"context\":\"current_turn\",\"effort\":\"low\",\"mode\":\"standard\",\"summary\":\"detailed\"},\"safety_identifier\":null,\"service_tier\":\"auto\",\"store\":false,\"temperature\":1,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tool_usage\":{\"image_gen\":{\"input_tokens\":0,\"input_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"output_tokens\":0,\"output_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"total_tokens\":0},\"web_search\":{\"num_requests\":0}},\"tools\":[{\"type\":\"function\",\"description\":\"Read a file or list a directory. For files, output is truncated to 2000 lines or 50KB — use offset/limit for large files. For directories, returns the list of entries.\",\"name\":\"read\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to read\"},\"offset\":{\"type\":\"number\",\"description\":\"Line number to start from (1-indexed)\"},\"limit\":{\"type\":\"number\",\"description\":\"Maximum number of lines to read\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Write content to a file. Creates the file and parent directories if they do not exist.\",\"name\":\"write\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"content\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to write\"},\"content\":{\"type\":\"string\",\"description\":\"Content to write to the file\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Edit a file using exact text replacement. The oldText must match a unique region of the file. Use replaceAll to replace all occurrences.\",\"name\":\"edit\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"oldText\",\"newText\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to edit\"},\"oldText\":{\"type\":\"string\",\"description\":\"Exact text to find (must be unique)\"},\"newText\":{\"type\":\"string\",\"description\":\"Replacement text\"},\"replaceAll\":{\"type\":\"boolean\",\"description\":\"Replace all occurrences\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Execute a bash command. Returns stdout and stderr. Output is truncated to the last 2000 lines or 50KB.\",\"name\":\"bash\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"command\"],\"properties\":{\"command\":{\"type\":\"string\",\"description\":\"Bash command to execute\"},\"timeout\":{\"type\":\"number\",\"description\":\"Timeout in seconds\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Search file contents for a regex pattern. Returns matching lines with file paths and line numbers.\",\"name\":\"grep\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Search pattern (regex)\"},\"path\":{\"type\":\"string\",\"description\":\"Directory or file to search (default: .)\"},\"include\":{\"type\":\"string\",\"description\":\"Glob filter, e.g. \\\"*.ts\\\"\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Find files by filename pattern using shell find -name semantics. Returns matching file paths.\",\"name\":\"glob\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Filename pattern, e.g. \\\"*.ts\\\"\"},\"path\":{\"type\":\"string\",\"description\":\"Directory to search in (default: .)\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Delegate a focused task to a detached child agent with its own context. Use this for independent research, file exploration, or parallel work. The task returns only its final answer to this conversation. No subagents are currently defined.\",\"name\":\"task\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"prompt\"],\"properties\":{\"description\":{\"type\":\"string\",\"description\":\"Short human-readable label for the delegated work\"},\"prompt\":{\"type\":\"string\",\"description\":\"Focused instructions for the child agent\"},\"agent\":{\"type\":\"string\",\"description\":\"Declared subagent to use for the child agent\"},\"cwd\":{\"type\":\"string\",\"description\":\"Working directory for the child agent. AGENTS.md and skills are discovered from here.\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Return a deterministic lookup result with an id needed by web_search.\",\"name\":\"lookup\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"query\"],\"properties\":{\"query\":{\"type\":\"string\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Search a deterministic local web index. Requires the id returned by lookup.\",\"name\":\"web_search\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"lookupId\",\"query\"],\"properties\":{\"lookupId\":{\"type\":\"string\"},\"query\":{\"type\":\"string\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Summarize the selected deterministic source after web_search returns a URL.\",\"name\":\"summarize_source\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"url\"],\"properties\":{\"url\":{\"type\":\"string\"}}},\"strict\":false}],\"top_logprobs\":0,\"top_p\":0.98,\"truncation\":\"disabled\",\"usage\":null,\"user\":null,\"metadata\":{}},\"sequence_number\":0}", - "event: response.in_progress\ndata: {\"type\":\"response.in_progress\",\"response\":{\"id\":\"resp_045458052dc10379016a705b5f8ed48191ac5f4bcf9d7df429\",\"object\":\"response\",\"created_at\":1785748319,\"status\":\"in_progress\",\"background\":false,\"completed_at\":null,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":null,\"max_tool_calls\":null,\"model\":\"gpt-5.4-nano-2026-03-17\",\"moderation\":null,\"output\":[],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":\"workflow:instrumentation:01KZ3E9TWTAEK4P4P3F63HKAKC::default::ma\",\"prompt_cache_retention\":\"24h\",\"reasoning\":{\"context\":\"current_turn\",\"effort\":\"low\",\"mode\":\"standard\",\"summary\":\"detailed\"},\"safety_identifier\":null,\"service_tier\":\"auto\",\"store\":false,\"temperature\":1,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tool_usage\":{\"image_gen\":{\"input_tokens\":0,\"input_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"output_tokens\":0,\"output_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"total_tokens\":0},\"web_search\":{\"num_requests\":0}},\"tools\":[{\"type\":\"function\",\"description\":\"Read a file or list a directory. For files, output is truncated to 2000 lines or 50KB — use offset/limit for large files. For directories, returns the list of entries.\",\"name\":\"read\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to read\"},\"offset\":{\"type\":\"number\",\"description\":\"Line number to start from (1-indexed)\"},\"limit\":{\"type\":\"number\",\"description\":\"Maximum number of lines to read\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Write content to a file. Creates the file and parent directories if they do not exist.\",\"name\":\"write\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"content\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to write\"},\"content\":{\"type\":\"string\",\"description\":\"Content to write to the file\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Edit a file using exact text replacement. The oldText must match a unique region of the file. Use replaceAll to replace all occurrences.\",\"name\":\"edit\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"oldText\",\"newText\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to edit\"},\"oldText\":{\"type\":\"string\",\"description\":\"Exact text to find (must be unique)\"},\"newText\":{\"type\":\"string\",\"description\":\"Replacement text\"},\"replaceAll\":{\"type\":\"boolean\",\"description\":\"Replace all occurrences\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Execute a bash command. Returns stdout and stderr. Output is truncated to the last 2000 lines or 50KB.\",\"name\":\"bash\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"command\"],\"properties\":{\"command\":{\"type\":\"string\",\"description\":\"Bash command to execute\"},\"timeout\":{\"type\":\"number\",\"description\":\"Timeout in seconds\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Search file contents for a regex pattern. Returns matching lines with file paths and line numbers.\",\"name\":\"grep\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Search pattern (regex)\"},\"path\":{\"type\":\"string\",\"description\":\"Directory or file to search (default: .)\"},\"include\":{\"type\":\"string\",\"description\":\"Glob filter, e.g. \\\"*.ts\\\"\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Find files by filename pattern using shell find -name semantics. Returns matching file paths.\",\"name\":\"glob\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Filename pattern, e.g. \\\"*.ts\\\"\"},\"path\":{\"type\":\"string\",\"description\":\"Directory to search in (default: .)\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Delegate a focused task to a detached child agent with its own context. Use this for independent research, file exploration, or parallel work. The task returns only its final answer to this conversation. No subagents are currently defined.\",\"name\":\"task\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"prompt\"],\"properties\":{\"description\":{\"type\":\"string\",\"description\":\"Short human-readable label for the delegated work\"},\"prompt\":{\"type\":\"string\",\"description\":\"Focused instructions for the child agent\"},\"agent\":{\"type\":\"string\",\"description\":\"Declared subagent to use for the child agent\"},\"cwd\":{\"type\":\"string\",\"description\":\"Working directory for the child agent. AGENTS.md and skills are discovered from here.\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Return a deterministic lookup result with an id needed by web_search.\",\"name\":\"lookup\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"query\"],\"properties\":{\"query\":{\"type\":\"string\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Search a deterministic local web index. Requires the id returned by lookup.\",\"name\":\"web_search\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"lookupId\",\"query\"],\"properties\":{\"lookupId\":{\"type\":\"string\"},\"query\":{\"type\":\"string\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Summarize the selected deterministic source after web_search returns a URL.\",\"name\":\"summarize_source\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"url\"],\"properties\":{\"url\":{\"type\":\"string\"}}},\"strict\":false}],\"top_logprobs\":0,\"top_p\":0.98,\"truncation\":\"disabled\",\"usage\":null,\"user\":null,\"metadata\":{}},\"sequence_number\":1}", - "event: response.output_item.added\ndata: {\"type\":\"response.output_item.added\",\"item\":{\"id\":\"fc_045458052dc10379016a705b6008408191b27653755cf5bd03\",\"type\":\"function_call\",\"status\":\"in_progress\",\"arguments\":\"\",\"call_id\":\"call_rBgonKxWHwhN9E8aQ4VobIOe\",\"name\":\"lookup\"},\"output_index\":0,\"sequence_number\":2}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"{\\\"\",\"item_id\":\"fc_045458052dc10379016a705b6008408191b27653755cf5bd03\",\"obfuscation\":\"FJ9TowjZDi2V7k\",\"output_index\":0,\"sequence_number\":3}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"query\",\"item_id\":\"fc_045458052dc10379016a705b6008408191b27653755cf5bd03\",\"obfuscation\":\"s6VEwiE9UsM\",\"output_index\":0,\"sequence_number\":4}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"\\\":\\\"\",\"item_id\":\"fc_045458052dc10379016a705b6008408191b27653755cf5bd03\",\"obfuscation\":\"6aWQMwZgQPiRf\",\"output_index\":0,\"sequence_number\":5}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"fl\",\"item_id\":\"fc_045458052dc10379016a705b6008408191b27653755cf5bd03\",\"obfuscation\":\"KUoAPUAlQ7cmX9\",\"output_index\":0,\"sequence_number\":6}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"ue\",\"item_id\":\"fc_045458052dc10379016a705b6008408191b27653755cf5bd03\",\"obfuscation\":\"lQUgpuDVqWyLbo\",\"output_index\":0,\"sequence_number\":7}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" instrumentation\",\"item_id\":\"fc_045458052dc10379016a705b6008408191b27653755cf5bd03\",\"obfuscation\":\"\",\"output_index\":0,\"sequence_number\":8}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"\\\"}\",\"item_id\":\"fc_045458052dc10379016a705b6008408191b27653755cf5bd03\",\"obfuscation\":\"jkQ7s9o73nxqwB\",\"output_index\":0,\"sequence_number\":9}", - "event: response.function_call_arguments.done\ndata: {\"type\":\"response.function_call_arguments.done\",\"arguments\":\"{\\\"query\\\":\\\"flue instrumentation\\\"}\",\"item_id\":\"fc_045458052dc10379016a705b6008408191b27653755cf5bd03\",\"output_index\":0,\"sequence_number\":10}", - "event: response.output_item.done\ndata: {\"type\":\"response.output_item.done\",\"item\":{\"id\":\"fc_045458052dc10379016a705b6008408191b27653755cf5bd03\",\"type\":\"function_call\",\"status\":\"completed\",\"arguments\":\"{\\\"query\\\":\\\"flue instrumentation\\\"}\",\"call_id\":\"call_rBgonKxWHwhN9E8aQ4VobIOe\",\"name\":\"lookup\"},\"output_index\":0,\"sequence_number\":11}", - "event: response.completed\ndata: {\"type\":\"response.completed\",\"response\":{\"id\":\"resp_045458052dc10379016a705b5f8ed48191ac5f4bcf9d7df429\",\"object\":\"response\",\"created_at\":1785748319,\"status\":\"completed\",\"background\":false,\"completed_at\":1785748320,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":null,\"max_tool_calls\":null,\"model\":\"gpt-5.4-nano-2026-03-17\",\"moderation\":null,\"output\":[{\"id\":\"fc_045458052dc10379016a705b6008408191b27653755cf5bd03\",\"type\":\"function_call\",\"status\":\"completed\",\"arguments\":\"{\\\"query\\\":\\\"flue instrumentation\\\"}\",\"call_id\":\"call_rBgonKxWHwhN9E8aQ4VobIOe\",\"name\":\"lookup\"}],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":\"workflow:instrumentation:01KZ3E9TWTAEK4P4P3F63HKAKC::default::ma\",\"prompt_cache_retention\":\"24h\",\"reasoning\":{\"context\":\"current_turn\",\"effort\":\"low\",\"mode\":\"standard\",\"summary\":\"detailed\"},\"safety_identifier\":null,\"service_tier\":\"default\",\"store\":false,\"temperature\":1,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tool_usage\":{\"image_gen\":{\"input_tokens\":0,\"input_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"output_tokens\":0,\"output_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"total_tokens\":0},\"web_search\":{\"num_requests\":0}},\"tools\":[{\"type\":\"function\",\"description\":\"Read a file or list a directory. For files, output is truncated to 2000 lines or 50KB — use offset/limit for large files. For directories, returns the list of entries.\",\"name\":\"read\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to read\"},\"offset\":{\"type\":\"number\",\"description\":\"Line number to start from (1-indexed)\"},\"limit\":{\"type\":\"number\",\"description\":\"Maximum number of lines to read\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Write content to a file. Creates the file and parent directories if they do not exist.\",\"name\":\"write\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"content\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to write\"},\"content\":{\"type\":\"string\",\"description\":\"Content to write to the file\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Edit a file using exact text replacement. The oldText must match a unique region of the file. Use replaceAll to replace all occurrences.\",\"name\":\"edit\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"oldText\",\"newText\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to edit\"},\"oldText\":{\"type\":\"string\",\"description\":\"Exact text to find (must be unique)\"},\"newText\":{\"type\":\"string\",\"description\":\"Replacement text\"},\"replaceAll\":{\"type\":\"boolean\",\"description\":\"Replace all occurrences\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Execute a bash command. Returns stdout and stderr. Output is truncated to the last 2000 lines or 50KB.\",\"name\":\"bash\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"command\"],\"properties\":{\"command\":{\"type\":\"string\",\"description\":\"Bash command to execute\"},\"timeout\":{\"type\":\"number\",\"description\":\"Timeout in seconds\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Search file contents for a regex pattern. Returns matching lines with file paths and line numbers.\",\"name\":\"grep\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Search pattern (regex)\"},\"path\":{\"type\":\"string\",\"description\":\"Directory or file to search (default: .)\"},\"include\":{\"type\":\"string\",\"description\":\"Glob filter, e.g. \\\"*.ts\\\"\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Find files by filename pattern using shell find -name semantics. Returns matching file paths.\",\"name\":\"glob\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Filename pattern, e.g. \\\"*.ts\\\"\"},\"path\":{\"type\":\"string\",\"description\":\"Directory to search in (default: .)\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Delegate a focused task to a detached child agent with its own context. Use this for independent research, file exploration, or parallel work. The task returns only its final answer to this conversation. No subagents are currently defined.\",\"name\":\"task\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"prompt\"],\"properties\":{\"description\":{\"type\":\"string\",\"description\":\"Short human-readable label for the delegated work\"},\"prompt\":{\"type\":\"string\",\"description\":\"Focused instructions for the child agent\"},\"agent\":{\"type\":\"string\",\"description\":\"Declared subagent to use for the child agent\"},\"cwd\":{\"type\":\"string\",\"description\":\"Working directory for the child agent. AGENTS.md and skills are discovered from here.\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Return a deterministic lookup result with an id needed by web_search.\",\"name\":\"lookup\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"query\"],\"properties\":{\"query\":{\"type\":\"string\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Search a deterministic local web index. Requires the id returned by lookup.\",\"name\":\"web_search\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"lookupId\",\"query\"],\"properties\":{\"lookupId\":{\"type\":\"string\"},\"query\":{\"type\":\"string\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Summarize the selected deterministic source after web_search returns a URL.\",\"name\":\"summarize_source\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"url\"],\"properties\":{\"url\":{\"type\":\"string\"}}},\"strict\":false}],\"top_logprobs\":0,\"top_p\":0.98,\"truncation\":\"disabled\",\"usage\":{\"input_tokens\":1005,\"input_tokens_details\":{\"cache_write_tokens\":0,\"cached_tokens\":0},\"output_tokens\":19,\"output_tokens_details\":{\"reasoning_tokens\":0},\"total_tokens\":1024},\"user\":null,\"metadata\":{}},\"sequence_number\":12}" - ], - "kind": "sse" - }, - "headers": { - "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", - "alt-svc": "h3=\":443\"; ma=86400", - "cf-cache-status": "DYNAMIC", - "cf-ray": "a25432b38d617bb6-IAD", - "connection": "keep-alive", - "content-type": "text/event-stream; charset=utf-8", - "date": "Mon, 03 Aug 2026 09:11:59 GMT", - "openai-organization": "braintrust-data", - "openai-processing-ms": "572", - "openai-project": "proj_vsCSXafhhByzWOThMrJcZiw9", - "openai-version": "2020-10-01", - "server": "cloudflare", - "set-cookie": "[REDACTED]", - "strict-transport-security": "max-age=31536000; includeSubDomains; preload", - "transfer-encoding": "chunked", - "x-content-type-options": "nosniff", - "x-ratelimit-limit-requests": "30000", - "x-ratelimit-limit-tokens": "180000000", - "x-ratelimit-remaining-requests": "29999", - "x-ratelimit-remaining-tokens": "179998539", - "x-ratelimit-reset-requests": "2ms", - "x-ratelimit-reset-tokens": "0s", - "x-request-id": "req_3ad47b9556ec41a8bf046bb73633b660" - }, - "status": 200, - "statusText": "OK" - } - }, - { - "callIndex": 1, - "id": "f12aff3913439248", - "matchKey": "POST api.openai.com/v1/responses", - "recordedAt": "2026-08-03T09:12:01.038Z", - "request": { - "body": { - "kind": "json", - "value": { - "include": ["reasoning.encrypted_content"], - "input": [ - { - "content": "You are running in headless mode with no human operator. Work autonomously — never ask questions, never wait for user input. Make your best judgment and proceed independently.\n\nYou are a deterministic Flue instrumentation test agent. Follow user instructions exactly. When asked for a marker, output only that marker and no extra text. When running a local skill file, read it yourself and do not delegate it to a task.\n\n## Available Skills\n\nThe following skills provide specialized instructions for specific tasks. When a task matches a skill description, activate that skill before proceeding so its full instructions are loaded. Skill instructions and supporting resources stay lazy until activation or explicit file reads.\n\n- **e2e-flue-skill** — Deterministic skill used by the Braintrust Flue instrumentation e2e test.\n\nDate: Mon, Aug 3, 2026\nWorking directory: .\n\nDirectory structure:\n.agents\n.flue\n.flue-build\n__cassettes__\n__snapshots__\nassertions.ts\ncassette-filter.mjs\nconstants.mjs\ndist\nnode_modules\npackage.json\npnpm-lock.yaml\nscenario.cli.mjs\nscenario.impl.mjs\nscenario.mjs\nscenario.test.ts\nscenario.ts\nscenario.v2.ts\nv2-assertions.ts\nversions", - "role": "developer" - }, - { - "content": [ - { - "text": "Complete this instrumented research flow. Call exactly one tool per turn and wait for each tool result before choosing the next tool. Step 1: call lookup with query \"flue instrumentation\". Step 2: use the lookup result id as lookupId and call web_search with query \"Braintrust Flue reasoning stream instrumentation\". Step 3: use the first web_search result url and call summarize_source. After summarize_source returns, reply with exactly PROMPT_DONE and no other text.", - "type": "input_text" - } - ], - "role": "user" - }, - { - "arguments": "{\"query\":\"flue instrumentation\"}", - "call_id": "call_rBgonKxWHwhN9E8aQ4VobIOe", - "id": "fc_045458052dc10379016a705b6008408191b27653755cf5bd03", - "name": "lookup", - "type": "function_call" - }, - { - "call_id": "call_rBgonKxWHwhN9E8aQ4VobIOe", - "output": "{\"id\":\"flue-session-2026\",\"query\":\"flue instrumentation\",\"topic\":\"session instrumentation\"}", - "type": "function_call_output" - } - ], - "model": "gpt-5.4-nano", - "prompt_cache_key": "[REDACTED]", - "reasoning": { - "effort": "low", - "summary": "auto" - }, - "store": false, - "stream": true, - "tools": [ - { - "description": "Read a file or list a directory. For files, output is truncated to 2000 lines or 50KB — use offset/limit for large files. For directories, returns the list of entries.", - "name": "read", - "parameters": { - "properties": { - "limit": { - "description": "Maximum number of lines to read", - "type": "number" - }, - "offset": { - "description": "Line number to start from (1-indexed)", - "type": "number" - }, - "path": { - "description": "Path to the file to read", - "type": "string" - } - }, - "required": ["path"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Write content to a file. Creates the file and parent directories if they do not exist.", - "name": "write", - "parameters": { - "properties": { - "content": { - "description": "Content to write to the file", - "type": "string" - }, - "path": { - "description": "Path to the file to write", - "type": "string" - } - }, - "required": ["path", "content"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Edit a file using exact text replacement. The oldText must match a unique region of the file. Use replaceAll to replace all occurrences.", - "name": "edit", - "parameters": { - "properties": { - "newText": { - "description": "Replacement text", - "type": "string" - }, - "oldText": { - "description": "Exact text to find (must be unique)", - "type": "string" - }, - "path": { - "description": "Path to the file to edit", - "type": "string" - }, - "replaceAll": { - "description": "Replace all occurrences", - "type": "boolean" - } - }, - "required": ["path", "oldText", "newText"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Execute a bash command. Returns stdout and stderr. Output is truncated to the last 2000 lines or 50KB.", - "name": "bash", - "parameters": { - "properties": { - "command": { - "description": "Bash command to execute", - "type": "string" - }, - "timeout": { - "description": "Timeout in seconds", - "type": "number" - } - }, - "required": ["command"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Search file contents for a regex pattern. Returns matching lines with file paths and line numbers.", - "name": "grep", - "parameters": { - "properties": { - "include": { - "description": "Glob filter, e.g. \"*.ts\"", - "type": "string" - }, - "path": { - "description": "Directory or file to search (default: .)", - "type": "string" - }, - "pattern": { - "description": "Search pattern (regex)", - "type": "string" - } - }, - "required": ["pattern"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Find files by filename pattern using shell find -name semantics. Returns matching file paths.", - "name": "glob", - "parameters": { - "properties": { - "path": { - "description": "Directory to search in (default: .)", - "type": "string" - }, - "pattern": { - "description": "Filename pattern, e.g. \"*.ts\"", - "type": "string" - } - }, - "required": ["pattern"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Delegate a focused task to a detached child agent with its own context. Use this for independent research, file exploration, or parallel work. The task returns only its final answer to this conversation. No subagents are currently defined.", - "name": "task", - "parameters": { - "properties": { - "agent": { - "description": "Declared subagent to use for the child agent", - "type": "string" - }, - "cwd": { - "description": "Working directory for the child agent. AGENTS.md and skills are discovered from here.", - "type": "string" - }, - "description": { - "description": "Short human-readable label for the delegated work", - "type": "string" - }, - "prompt": { - "description": "Focused instructions for the child agent", - "type": "string" - } - }, - "required": ["prompt"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Return a deterministic lookup result with an id needed by web_search.", - "name": "lookup", - "parameters": { - "properties": { - "query": { - "type": "string" - } - }, - "required": ["query"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Search a deterministic local web index. Requires the id returned by lookup.", - "name": "web_search", - "parameters": { - "properties": { - "lookupId": { - "type": "string" - }, - "query": { - "type": "string" - } - }, - "required": ["lookupId", "query"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Summarize the selected deterministic source after web_search returns a URL.", - "name": "summarize_source", - "parameters": { - "properties": { - "url": { - "type": "string" - } - }, - "required": ["url"], - "type": "object" - }, - "strict": false, - "type": "function" - } - ] - } - }, - "headers": {}, - "method": "POST", - "url": "https://api.openai.com/v1/responses" - }, - "response": { - "body": { - "chunks": [ - "event: response.created\ndata: {\"type\":\"response.created\",\"response\":{\"id\":\"resp_045458052dc10379016a705b6070c48191b73ad8a8f9b550f7\",\"object\":\"response\",\"created_at\":1785748320,\"status\":\"in_progress\",\"background\":false,\"completed_at\":null,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":null,\"max_tool_calls\":null,\"model\":\"gpt-5.4-nano-2026-03-17\",\"moderation\":null,\"output\":[],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":\"workflow:instrumentation:01KZ3E9TWTAEK4P4P3F63HKAKC::default::ma\",\"prompt_cache_retention\":\"24h\",\"reasoning\":{\"context\":\"current_turn\",\"effort\":\"low\",\"mode\":\"standard\",\"summary\":\"detailed\"},\"safety_identifier\":null,\"service_tier\":\"auto\",\"store\":false,\"temperature\":1,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tool_usage\":{\"image_gen\":{\"input_tokens\":0,\"input_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"output_tokens\":0,\"output_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"total_tokens\":0},\"web_search\":{\"num_requests\":0}},\"tools\":[{\"type\":\"function\",\"description\":\"Read a file or list a directory. For files, output is truncated to 2000 lines or 50KB — use offset/limit for large files. For directories, returns the list of entries.\",\"name\":\"read\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to read\"},\"offset\":{\"type\":\"number\",\"description\":\"Line number to start from (1-indexed)\"},\"limit\":{\"type\":\"number\",\"description\":\"Maximum number of lines to read\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Write content to a file. Creates the file and parent directories if they do not exist.\",\"name\":\"write\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"content\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to write\"},\"content\":{\"type\":\"string\",\"description\":\"Content to write to the file\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Edit a file using exact text replacement. The oldText must match a unique region of the file. Use replaceAll to replace all occurrences.\",\"name\":\"edit\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"oldText\",\"newText\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to edit\"},\"oldText\":{\"type\":\"string\",\"description\":\"Exact text to find (must be unique)\"},\"newText\":{\"type\":\"string\",\"description\":\"Replacement text\"},\"replaceAll\":{\"type\":\"boolean\",\"description\":\"Replace all occurrences\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Execute a bash command. Returns stdout and stderr. Output is truncated to the last 2000 lines or 50KB.\",\"name\":\"bash\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"command\"],\"properties\":{\"command\":{\"type\":\"string\",\"description\":\"Bash command to execute\"},\"timeout\":{\"type\":\"number\",\"description\":\"Timeout in seconds\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Search file contents for a regex pattern. Returns matching lines with file paths and line numbers.\",\"name\":\"grep\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Search pattern (regex)\"},\"path\":{\"type\":\"string\",\"description\":\"Directory or file to search (default: .)\"},\"include\":{\"type\":\"string\",\"description\":\"Glob filter, e.g. \\\"*.ts\\\"\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Find files by filename pattern using shell find -name semantics. Returns matching file paths.\",\"name\":\"glob\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Filename pattern, e.g. \\\"*.ts\\\"\"},\"path\":{\"type\":\"string\",\"description\":\"Directory to search in (default: .)\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Delegate a focused task to a detached child agent with its own context. Use this for independent research, file exploration, or parallel work. The task returns only its final answer to this conversation. No subagents are currently defined.\",\"name\":\"task\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"prompt\"],\"properties\":{\"description\":{\"type\":\"string\",\"description\":\"Short human-readable label for the delegated work\"},\"prompt\":{\"type\":\"string\",\"description\":\"Focused instructions for the child agent\"},\"agent\":{\"type\":\"string\",\"description\":\"Declared subagent to use for the child agent\"},\"cwd\":{\"type\":\"string\",\"description\":\"Working directory for the child agent. AGENTS.md and skills are discovered from here.\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Return a deterministic lookup result with an id needed by web_search.\",\"name\":\"lookup\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"query\"],\"properties\":{\"query\":{\"type\":\"string\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Search a deterministic local web index. Requires the id returned by lookup.\",\"name\":\"web_search\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"lookupId\",\"query\"],\"properties\":{\"lookupId\":{\"type\":\"string\"},\"query\":{\"type\":\"string\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Summarize the selected deterministic source after web_search returns a URL.\",\"name\":\"summarize_source\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"url\"],\"properties\":{\"url\":{\"type\":\"string\"}}},\"strict\":false}],\"top_logprobs\":0,\"top_p\":0.98,\"truncation\":\"disabled\",\"usage\":null,\"user\":null,\"metadata\":{}},\"sequence_number\":0}", - "event: response.in_progress\ndata: {\"type\":\"response.in_progress\",\"response\":{\"id\":\"resp_045458052dc10379016a705b6070c48191b73ad8a8f9b550f7\",\"object\":\"response\",\"created_at\":1785748320,\"status\":\"in_progress\",\"background\":false,\"completed_at\":null,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":null,\"max_tool_calls\":null,\"model\":\"gpt-5.4-nano-2026-03-17\",\"moderation\":null,\"output\":[],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":\"workflow:instrumentation:01KZ3E9TWTAEK4P4P3F63HKAKC::default::ma\",\"prompt_cache_retention\":\"24h\",\"reasoning\":{\"context\":\"current_turn\",\"effort\":\"low\",\"mode\":\"standard\",\"summary\":\"detailed\"},\"safety_identifier\":null,\"service_tier\":\"auto\",\"store\":false,\"temperature\":1,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tool_usage\":{\"image_gen\":{\"input_tokens\":0,\"input_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"output_tokens\":0,\"output_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"total_tokens\":0},\"web_search\":{\"num_requests\":0}},\"tools\":[{\"type\":\"function\",\"description\":\"Read a file or list a directory. For files, output is truncated to 2000 lines or 50KB — use offset/limit for large files. For directories, returns the list of entries.\",\"name\":\"read\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to read\"},\"offset\":{\"type\":\"number\",\"description\":\"Line number to start from (1-indexed)\"},\"limit\":{\"type\":\"number\",\"description\":\"Maximum number of lines to read\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Write content to a file. Creates the file and parent directories if they do not exist.\",\"name\":\"write\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"content\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to write\"},\"content\":{\"type\":\"string\",\"description\":\"Content to write to the file\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Edit a file using exact text replacement. The oldText must match a unique region of the file. Use replaceAll to replace all occurrences.\",\"name\":\"edit\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"oldText\",\"newText\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to edit\"},\"oldText\":{\"type\":\"string\",\"description\":\"Exact text to find (must be unique)\"},\"newText\":{\"type\":\"string\",\"description\":\"Replacement text\"},\"replaceAll\":{\"type\":\"boolean\",\"description\":\"Replace all occurrences\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Execute a bash command. Returns stdout and stderr. Output is truncated to the last 2000 lines or 50KB.\",\"name\":\"bash\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"command\"],\"properties\":{\"command\":{\"type\":\"string\",\"description\":\"Bash command to execute\"},\"timeout\":{\"type\":\"number\",\"description\":\"Timeout in seconds\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Search file contents for a regex pattern. Returns matching lines with file paths and line numbers.\",\"name\":\"grep\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Search pattern (regex)\"},\"path\":{\"type\":\"string\",\"description\":\"Directory or file to search (default: .)\"},\"include\":{\"type\":\"string\",\"description\":\"Glob filter, e.g. \\\"*.ts\\\"\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Find files by filename pattern using shell find -name semantics. Returns matching file paths.\",\"name\":\"glob\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Filename pattern, e.g. \\\"*.ts\\\"\"},\"path\":{\"type\":\"string\",\"description\":\"Directory to search in (default: .)\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Delegate a focused task to a detached child agent with its own context. Use this for independent research, file exploration, or parallel work. The task returns only its final answer to this conversation. No subagents are currently defined.\",\"name\":\"task\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"prompt\"],\"properties\":{\"description\":{\"type\":\"string\",\"description\":\"Short human-readable label for the delegated work\"},\"prompt\":{\"type\":\"string\",\"description\":\"Focused instructions for the child agent\"},\"agent\":{\"type\":\"string\",\"description\":\"Declared subagent to use for the child agent\"},\"cwd\":{\"type\":\"string\",\"description\":\"Working directory for the child agent. AGENTS.md and skills are discovered from here.\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Return a deterministic lookup result with an id needed by web_search.\",\"name\":\"lookup\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"query\"],\"properties\":{\"query\":{\"type\":\"string\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Search a deterministic local web index. Requires the id returned by lookup.\",\"name\":\"web_search\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"lookupId\",\"query\"],\"properties\":{\"lookupId\":{\"type\":\"string\"},\"query\":{\"type\":\"string\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Summarize the selected deterministic source after web_search returns a URL.\",\"name\":\"summarize_source\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"url\"],\"properties\":{\"url\":{\"type\":\"string\"}}},\"strict\":false}],\"top_logprobs\":0,\"top_p\":0.98,\"truncation\":\"disabled\",\"usage\":null,\"user\":null,\"metadata\":{}},\"sequence_number\":1}", - "event: response.output_item.added\ndata: {\"type\":\"response.output_item.added\",\"item\":{\"id\":\"fc_045458052dc10379016a705b60c92c81918da2fab9710ee199\",\"type\":\"function_call\",\"status\":\"in_progress\",\"arguments\":\"\",\"call_id\":\"call_fGesVCba2aABMoxSNfVnDVnR\",\"name\":\"web_search\"},\"output_index\":0,\"sequence_number\":2}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"{\\\"\",\"item_id\":\"fc_045458052dc10379016a705b60c92c81918da2fab9710ee199\",\"obfuscation\":\"CXnKLapUymev31\",\"output_index\":0,\"sequence_number\":3}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"lookup\",\"item_id\":\"fc_045458052dc10379016a705b60c92c81918da2fab9710ee199\",\"obfuscation\":\"hPsgf6WVxe\",\"output_index\":0,\"sequence_number\":4}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"Id\",\"item_id\":\"fc_045458052dc10379016a705b60c92c81918da2fab9710ee199\",\"obfuscation\":\"jPh8SwIrBbqksW\",\"output_index\":0,\"sequence_number\":5}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"\\\":\\\"\",\"item_id\":\"fc_045458052dc10379016a705b60c92c81918da2fab9710ee199\",\"obfuscation\":\"IlU35DSXGkesV\",\"output_index\":0,\"sequence_number\":6}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"fl\",\"item_id\":\"fc_045458052dc10379016a705b60c92c81918da2fab9710ee199\",\"obfuscation\":\"1XJ1BKxdLr8xZ1\",\"output_index\":0,\"sequence_number\":7}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"ue\",\"item_id\":\"fc_045458052dc10379016a705b60c92c81918da2fab9710ee199\",\"obfuscation\":\"1OTUMDyCdYkVv8\",\"output_index\":0,\"sequence_number\":8}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"-session\",\"item_id\":\"fc_045458052dc10379016a705b60c92c81918da2fab9710ee199\",\"obfuscation\":\"Nuz4FnUt\",\"output_index\":0,\"sequence_number\":9}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"-\",\"item_id\":\"fc_045458052dc10379016a705b60c92c81918da2fab9710ee199\",\"obfuscation\":\"HwzHAjBygsfiSUX\",\"output_index\":0,\"sequence_number\":10}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"202\",\"item_id\":\"fc_045458052dc10379016a705b60c92c81918da2fab9710ee199\",\"obfuscation\":\"Yp4H7ZJmIH7Dv\",\"output_index\":0,\"sequence_number\":11}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"6\",\"item_id\":\"fc_045458052dc10379016a705b60c92c81918da2fab9710ee199\",\"obfuscation\":\"EYhWYsmGpl4z0l3\",\"output_index\":0,\"sequence_number\":12}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"\\\",\\\"\",\"item_id\":\"fc_045458052dc10379016a705b60c92c81918da2fab9710ee199\",\"obfuscation\":\"8RFEMjEMC5stu\",\"output_index\":0,\"sequence_number\":13}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"query\",\"item_id\":\"fc_045458052dc10379016a705b60c92c81918da2fab9710ee199\",\"obfuscation\":\"1ZkQEmh5ANn\",\"output_index\":0,\"sequence_number\":14}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"\\\":\\\"\",\"item_id\":\"fc_045458052dc10379016a705b60c92c81918da2fab9710ee199\",\"obfuscation\":\"0KsiqU7uz6ecQ\",\"output_index\":0,\"sequence_number\":15}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"Brain\",\"item_id\":\"fc_045458052dc10379016a705b60c92c81918da2fab9710ee199\",\"obfuscation\":\"nrOMFbiD09E\",\"output_index\":0,\"sequence_number\":16}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"trust\",\"item_id\":\"fc_045458052dc10379016a705b60c92c81918da2fab9710ee199\",\"obfuscation\":\"QnKfskbAhz0\",\"output_index\":0,\"sequence_number\":17}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" Fl\",\"item_id\":\"fc_045458052dc10379016a705b60c92c81918da2fab9710ee199\",\"obfuscation\":\"q6Uz9SRPWbTCf\",\"output_index\":0,\"sequence_number\":18}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"ue\",\"item_id\":\"fc_045458052dc10379016a705b60c92c81918da2fab9710ee199\",\"obfuscation\":\"ZFMpp0BC39uzCN\",\"output_index\":0,\"sequence_number\":19}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" reasoning\",\"item_id\":\"fc_045458052dc10379016a705b60c92c81918da2fab9710ee199\",\"obfuscation\":\"2MwmX9\",\"output_index\":0,\"sequence_number\":20}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" stream\",\"item_id\":\"fc_045458052dc10379016a705b60c92c81918da2fab9710ee199\",\"obfuscation\":\"2aJP5sGuM\",\"output_index\":0,\"sequence_number\":21}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" instrumentation\",\"item_id\":\"fc_045458052dc10379016a705b60c92c81918da2fab9710ee199\",\"obfuscation\":\"\",\"output_index\":0,\"sequence_number\":22}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"\\\"}\",\"item_id\":\"fc_045458052dc10379016a705b60c92c81918da2fab9710ee199\",\"obfuscation\":\"UvySj7uLIjgfiB\",\"output_index\":0,\"sequence_number\":23}", - "event: response.function_call_arguments.done\ndata: {\"type\":\"response.function_call_arguments.done\",\"arguments\":\"{\\\"lookupId\\\":\\\"flue-session-2026\\\",\\\"query\\\":\\\"Braintrust Flue reasoning stream instrumentation\\\"}\",\"item_id\":\"fc_045458052dc10379016a705b60c92c81918da2fab9710ee199\",\"output_index\":0,\"sequence_number\":24}", - "event: response.output_item.done\ndata: {\"type\":\"response.output_item.done\",\"item\":{\"id\":\"fc_045458052dc10379016a705b60c92c81918da2fab9710ee199\",\"type\":\"function_call\",\"status\":\"completed\",\"arguments\":\"{\\\"lookupId\\\":\\\"flue-session-2026\\\",\\\"query\\\":\\\"Braintrust Flue reasoning stream instrumentation\\\"}\",\"call_id\":\"call_fGesVCba2aABMoxSNfVnDVnR\",\"name\":\"web_search\"},\"output_index\":0,\"sequence_number\":25}", - "event: response.completed\ndata: {\"type\":\"response.completed\",\"response\":{\"id\":\"resp_045458052dc10379016a705b6070c48191b73ad8a8f9b550f7\",\"object\":\"response\",\"created_at\":1785748320,\"status\":\"completed\",\"background\":false,\"completed_at\":1785748320,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":null,\"max_tool_calls\":null,\"model\":\"gpt-5.4-nano-2026-03-17\",\"moderation\":null,\"output\":[{\"id\":\"fc_045458052dc10379016a705b60c92c81918da2fab9710ee199\",\"type\":\"function_call\",\"status\":\"completed\",\"arguments\":\"{\\\"lookupId\\\":\\\"flue-session-2026\\\",\\\"query\\\":\\\"Braintrust Flue reasoning stream instrumentation\\\"}\",\"call_id\":\"call_fGesVCba2aABMoxSNfVnDVnR\",\"name\":\"web_search\"}],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":\"workflow:instrumentation:01KZ3E9TWTAEK4P4P3F63HKAKC::default::ma\",\"prompt_cache_retention\":\"24h\",\"reasoning\":{\"context\":\"current_turn\",\"effort\":\"low\",\"mode\":\"standard\",\"summary\":\"detailed\"},\"safety_identifier\":null,\"service_tier\":\"default\",\"store\":false,\"temperature\":1,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tool_usage\":{\"image_gen\":{\"input_tokens\":0,\"input_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"output_tokens\":0,\"output_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"total_tokens\":0},\"web_search\":{\"num_requests\":0}},\"tools\":[{\"type\":\"function\",\"description\":\"Read a file or list a directory. For files, output is truncated to 2000 lines or 50KB — use offset/limit for large files. For directories, returns the list of entries.\",\"name\":\"read\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to read\"},\"offset\":{\"type\":\"number\",\"description\":\"Line number to start from (1-indexed)\"},\"limit\":{\"type\":\"number\",\"description\":\"Maximum number of lines to read\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Write content to a file. Creates the file and parent directories if they do not exist.\",\"name\":\"write\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"content\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to write\"},\"content\":{\"type\":\"string\",\"description\":\"Content to write to the file\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Edit a file using exact text replacement. The oldText must match a unique region of the file. Use replaceAll to replace all occurrences.\",\"name\":\"edit\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"oldText\",\"newText\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to edit\"},\"oldText\":{\"type\":\"string\",\"description\":\"Exact text to find (must be unique)\"},\"newText\":{\"type\":\"string\",\"description\":\"Replacement text\"},\"replaceAll\":{\"type\":\"boolean\",\"description\":\"Replace all occurrences\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Execute a bash command. Returns stdout and stderr. Output is truncated to the last 2000 lines or 50KB.\",\"name\":\"bash\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"command\"],\"properties\":{\"command\":{\"type\":\"string\",\"description\":\"Bash command to execute\"},\"timeout\":{\"type\":\"number\",\"description\":\"Timeout in seconds\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Search file contents for a regex pattern. Returns matching lines with file paths and line numbers.\",\"name\":\"grep\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Search pattern (regex)\"},\"path\":{\"type\":\"string\",\"description\":\"Directory or file to search (default: .)\"},\"include\":{\"type\":\"string\",\"description\":\"Glob filter, e.g. \\\"*.ts\\\"\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Find files by filename pattern using shell find -name semantics. Returns matching file paths.\",\"name\":\"glob\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Filename pattern, e.g. \\\"*.ts\\\"\"},\"path\":{\"type\":\"string\",\"description\":\"Directory to search in (default: .)\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Delegate a focused task to a detached child agent with its own context. Use this for independent research, file exploration, or parallel work. The task returns only its final answer to this conversation. No subagents are currently defined.\",\"name\":\"task\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"prompt\"],\"properties\":{\"description\":{\"type\":\"string\",\"description\":\"Short human-readable label for the delegated work\"},\"prompt\":{\"type\":\"string\",\"description\":\"Focused instructions for the child agent\"},\"agent\":{\"type\":\"string\",\"description\":\"Declared subagent to use for the child agent\"},\"cwd\":{\"type\":\"string\",\"description\":\"Working directory for the child agent. AGENTS.md and skills are discovered from here.\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Return a deterministic lookup result with an id needed by web_search.\",\"name\":\"lookup\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"query\"],\"properties\":{\"query\":{\"type\":\"string\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Search a deterministic local web index. Requires the id returned by lookup.\",\"name\":\"web_search\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"lookupId\",\"query\"],\"properties\":{\"lookupId\":{\"type\":\"string\"},\"query\":{\"type\":\"string\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Summarize the selected deterministic source after web_search returns a URL.\",\"name\":\"summarize_source\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"url\"],\"properties\":{\"url\":{\"type\":\"string\"}}},\"strict\":false}],\"top_logprobs\":0,\"top_p\":0.98,\"truncation\":\"disabled\",\"usage\":{\"input_tokens\":1055,\"input_tokens_details\":{\"cache_write_tokens\":0,\"cached_tokens\":0},\"output_tokens\":34,\"output_tokens_details\":{\"reasoning_tokens\":0},\"total_tokens\":1089},\"user\":null,\"metadata\":{}},\"sequence_number\":26}" - ], - "kind": "sse" - }, - "headers": { - "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", - "alt-svc": "h3=\":443\"; ma=86400", - "cf-cache-status": "DYNAMIC", - "cf-ray": "a25432ba7c1a7bb6-IAD", - "connection": "keep-alive", - "content-type": "text/event-stream; charset=utf-8", - "date": "Mon, 03 Aug 2026 09:12:00 GMT", - "openai-organization": "braintrust-data", - "openai-processing-ms": "236", - "openai-project": "proj_vsCSXafhhByzWOThMrJcZiw9", - "openai-version": "2020-10-01", - "server": "cloudflare", - "set-cookie": "[REDACTED]", - "strict-transport-security": "max-age=31536000; includeSubDomains; preload", - "transfer-encoding": "chunked", - "x-content-type-options": "nosniff", - "x-ratelimit-limit-requests": "30000", - "x-ratelimit-limit-tokens": "180000000", - "x-ratelimit-remaining-requests": "29999", - "x-ratelimit-remaining-tokens": "179998488", - "x-ratelimit-reset-requests": "2ms", - "x-ratelimit-reset-tokens": "0s", - "x-request-id": "req_b7babd52eb11438ca65da8e3e06cd733" - }, - "status": 200, - "statusText": "OK" - } - }, - { - "callIndex": 2, - "id": "5d2312f425df9f2f", - "matchKey": "POST api.openai.com/v1/responses", - "recordedAt": "2026-08-03T09:12:02.466Z", - "request": { - "body": { - "kind": "json", - "value": { - "include": ["reasoning.encrypted_content"], - "input": [ - { - "content": "You are running in headless mode with no human operator. Work autonomously — never ask questions, never wait for user input. Make your best judgment and proceed independently.\n\nYou are a deterministic Flue instrumentation test agent. Follow user instructions exactly. When asked for a marker, output only that marker and no extra text. When running a local skill file, read it yourself and do not delegate it to a task.\n\n## Available Skills\n\nThe following skills provide specialized instructions for specific tasks. When a task matches a skill description, activate that skill before proceeding so its full instructions are loaded. Skill instructions and supporting resources stay lazy until activation or explicit file reads.\n\n- **e2e-flue-skill** — Deterministic skill used by the Braintrust Flue instrumentation e2e test.\n\nDate: Mon, Aug 3, 2026\nWorking directory: .\n\nDirectory structure:\n.agents\n.flue\n.flue-build\n__cassettes__\n__snapshots__\nassertions.ts\ncassette-filter.mjs\nconstants.mjs\ndist\nnode_modules\npackage.json\npnpm-lock.yaml\nscenario.cli.mjs\nscenario.impl.mjs\nscenario.mjs\nscenario.test.ts\nscenario.ts\nscenario.v2.ts\nv2-assertions.ts\nversions", - "role": "developer" - }, - { - "content": [ - { - "text": "Complete this instrumented research flow. Call exactly one tool per turn and wait for each tool result before choosing the next tool. Step 1: call lookup with query \"flue instrumentation\". Step 2: use the lookup result id as lookupId and call web_search with query \"Braintrust Flue reasoning stream instrumentation\". Step 3: use the first web_search result url and call summarize_source. After summarize_source returns, reply with exactly PROMPT_DONE and no other text.", - "type": "input_text" - } - ], - "role": "user" - }, - { - "arguments": "{\"query\":\"flue instrumentation\"}", - "call_id": "call_rBgonKxWHwhN9E8aQ4VobIOe", - "id": "fc_045458052dc10379016a705b6008408191b27653755cf5bd03", - "name": "lookup", - "type": "function_call" - }, - { - "call_id": "call_rBgonKxWHwhN9E8aQ4VobIOe", - "output": "{\"id\":\"flue-session-2026\",\"query\":\"flue instrumentation\",\"topic\":\"session instrumentation\"}", - "type": "function_call_output" - }, - { - "arguments": "{\"lookupId\":\"flue-session-2026\",\"query\":\"Braintrust Flue reasoning stream instrumentation\"}", - "call_id": "call_fGesVCba2aABMoxSNfVnDVnR", - "id": "fc_045458052dc10379016a705b60c92c81918da2fab9710ee199", - "name": "web_search", - "type": "function_call" - }, - { - "call_id": "call_fGesVCba2aABMoxSNfVnDVnR", - "output": "{\"lookupId\":\"flue-session-2026\",\"query\":\"Braintrust Flue reasoning stream instrumentation\",\"results\":[{\"title\":\"Flue reasoning stream instrumentation\",\"url\":\"https://example.test/flue/reasoning-streams\"}]}", - "type": "function_call_output" - } - ], - "model": "gpt-5.4-nano", - "prompt_cache_key": "[REDACTED]", - "reasoning": { - "effort": "low", - "summary": "auto" - }, - "store": false, - "stream": true, - "tools": [ - { - "description": "Read a file or list a directory. For files, output is truncated to 2000 lines or 50KB — use offset/limit for large files. For directories, returns the list of entries.", - "name": "read", - "parameters": { - "properties": { - "limit": { - "description": "Maximum number of lines to read", - "type": "number" - }, - "offset": { - "description": "Line number to start from (1-indexed)", - "type": "number" - }, - "path": { - "description": "Path to the file to read", - "type": "string" - } - }, - "required": ["path"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Write content to a file. Creates the file and parent directories if they do not exist.", - "name": "write", - "parameters": { - "properties": { - "content": { - "description": "Content to write to the file", - "type": "string" - }, - "path": { - "description": "Path to the file to write", - "type": "string" - } - }, - "required": ["path", "content"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Edit a file using exact text replacement. The oldText must match a unique region of the file. Use replaceAll to replace all occurrences.", - "name": "edit", - "parameters": { - "properties": { - "newText": { - "description": "Replacement text", - "type": "string" - }, - "oldText": { - "description": "Exact text to find (must be unique)", - "type": "string" - }, - "path": { - "description": "Path to the file to edit", - "type": "string" - }, - "replaceAll": { - "description": "Replace all occurrences", - "type": "boolean" - } - }, - "required": ["path", "oldText", "newText"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Execute a bash command. Returns stdout and stderr. Output is truncated to the last 2000 lines or 50KB.", - "name": "bash", - "parameters": { - "properties": { - "command": { - "description": "Bash command to execute", - "type": "string" - }, - "timeout": { - "description": "Timeout in seconds", - "type": "number" - } - }, - "required": ["command"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Search file contents for a regex pattern. Returns matching lines with file paths and line numbers.", - "name": "grep", - "parameters": { - "properties": { - "include": { - "description": "Glob filter, e.g. \"*.ts\"", - "type": "string" - }, - "path": { - "description": "Directory or file to search (default: .)", - "type": "string" - }, - "pattern": { - "description": "Search pattern (regex)", - "type": "string" - } - }, - "required": ["pattern"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Find files by filename pattern using shell find -name semantics. Returns matching file paths.", - "name": "glob", - "parameters": { - "properties": { - "path": { - "description": "Directory to search in (default: .)", - "type": "string" - }, - "pattern": { - "description": "Filename pattern, e.g. \"*.ts\"", - "type": "string" - } - }, - "required": ["pattern"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Delegate a focused task to a detached child agent with its own context. Use this for independent research, file exploration, or parallel work. The task returns only its final answer to this conversation. No subagents are currently defined.", - "name": "task", - "parameters": { - "properties": { - "agent": { - "description": "Declared subagent to use for the child agent", - "type": "string" - }, - "cwd": { - "description": "Working directory for the child agent. AGENTS.md and skills are discovered from here.", - "type": "string" - }, - "description": { - "description": "Short human-readable label for the delegated work", - "type": "string" - }, - "prompt": { - "description": "Focused instructions for the child agent", - "type": "string" - } - }, - "required": ["prompt"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Return a deterministic lookup result with an id needed by web_search.", - "name": "lookup", - "parameters": { - "properties": { - "query": { - "type": "string" - } - }, - "required": ["query"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Search a deterministic local web index. Requires the id returned by lookup.", - "name": "web_search", - "parameters": { - "properties": { - "lookupId": { - "type": "string" - }, - "query": { - "type": "string" - } - }, - "required": ["lookupId", "query"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Summarize the selected deterministic source after web_search returns a URL.", - "name": "summarize_source", - "parameters": { - "properties": { - "url": { - "type": "string" - } - }, - "required": ["url"], - "type": "object" - }, - "strict": false, - "type": "function" - } - ] - } - }, - "headers": {}, - "method": "POST", - "url": "https://api.openai.com/v1/responses" - }, - "response": { - "body": { - "chunks": [ - "event: response.created\ndata: {\"type\":\"response.created\",\"response\":{\"id\":\"resp_045458052dc10379016a705b611e8881919ad2adde8d6874b9\",\"object\":\"response\",\"created_at\":1785748321,\"status\":\"in_progress\",\"background\":false,\"completed_at\":null,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":null,\"max_tool_calls\":null,\"model\":\"gpt-5.4-nano-2026-03-17\",\"moderation\":null,\"output\":[],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":\"workflow:instrumentation:01KZ3E9TWTAEK4P4P3F63HKAKC::default::ma\",\"prompt_cache_retention\":\"24h\",\"reasoning\":{\"context\":\"current_turn\",\"effort\":\"low\",\"mode\":\"standard\",\"summary\":\"detailed\"},\"safety_identifier\":null,\"service_tier\":\"auto\",\"store\":false,\"temperature\":1,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tool_usage\":{\"image_gen\":{\"input_tokens\":0,\"input_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"output_tokens\":0,\"output_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"total_tokens\":0},\"web_search\":{\"num_requests\":0}},\"tools\":[{\"type\":\"function\",\"description\":\"Read a file or list a directory. For files, output is truncated to 2000 lines or 50KB — use offset/limit for large files. For directories, returns the list of entries.\",\"name\":\"read\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to read\"},\"offset\":{\"type\":\"number\",\"description\":\"Line number to start from (1-indexed)\"},\"limit\":{\"type\":\"number\",\"description\":\"Maximum number of lines to read\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Write content to a file. Creates the file and parent directories if they do not exist.\",\"name\":\"write\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"content\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to write\"},\"content\":{\"type\":\"string\",\"description\":\"Content to write to the file\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Edit a file using exact text replacement. The oldText must match a unique region of the file. Use replaceAll to replace all occurrences.\",\"name\":\"edit\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"oldText\",\"newText\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to edit\"},\"oldText\":{\"type\":\"string\",\"description\":\"Exact text to find (must be unique)\"},\"newText\":{\"type\":\"string\",\"description\":\"Replacement text\"},\"replaceAll\":{\"type\":\"boolean\",\"description\":\"Replace all occurrences\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Execute a bash command. Returns stdout and stderr. Output is truncated to the last 2000 lines or 50KB.\",\"name\":\"bash\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"command\"],\"properties\":{\"command\":{\"type\":\"string\",\"description\":\"Bash command to execute\"},\"timeout\":{\"type\":\"number\",\"description\":\"Timeout in seconds\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Search file contents for a regex pattern. Returns matching lines with file paths and line numbers.\",\"name\":\"grep\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Search pattern (regex)\"},\"path\":{\"type\":\"string\",\"description\":\"Directory or file to search (default: .)\"},\"include\":{\"type\":\"string\",\"description\":\"Glob filter, e.g. \\\"*.ts\\\"\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Find files by filename pattern using shell find -name semantics. Returns matching file paths.\",\"name\":\"glob\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Filename pattern, e.g. \\\"*.ts\\\"\"},\"path\":{\"type\":\"string\",\"description\":\"Directory to search in (default: .)\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Delegate a focused task to a detached child agent with its own context. Use this for independent research, file exploration, or parallel work. The task returns only its final answer to this conversation. No subagents are currently defined.\",\"name\":\"task\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"prompt\"],\"properties\":{\"description\":{\"type\":\"string\",\"description\":\"Short human-readable label for the delegated work\"},\"prompt\":{\"type\":\"string\",\"description\":\"Focused instructions for the child agent\"},\"agent\":{\"type\":\"string\",\"description\":\"Declared subagent to use for the child agent\"},\"cwd\":{\"type\":\"string\",\"description\":\"Working directory for the child agent. AGENTS.md and skills are discovered from here.\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Return a deterministic lookup result with an id needed by web_search.\",\"name\":\"lookup\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"query\"],\"properties\":{\"query\":{\"type\":\"string\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Search a deterministic local web index. Requires the id returned by lookup.\",\"name\":\"web_search\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"lookupId\",\"query\"],\"properties\":{\"lookupId\":{\"type\":\"string\"},\"query\":{\"type\":\"string\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Summarize the selected deterministic source after web_search returns a URL.\",\"name\":\"summarize_source\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"url\"],\"properties\":{\"url\":{\"type\":\"string\"}}},\"strict\":false}],\"top_logprobs\":0,\"top_p\":0.98,\"truncation\":\"disabled\",\"usage\":null,\"user\":null,\"metadata\":{}},\"sequence_number\":0}", - "event: response.in_progress\ndata: {\"type\":\"response.in_progress\",\"response\":{\"id\":\"resp_045458052dc10379016a705b611e8881919ad2adde8d6874b9\",\"object\":\"response\",\"created_at\":1785748321,\"status\":\"in_progress\",\"background\":false,\"completed_at\":null,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":null,\"max_tool_calls\":null,\"model\":\"gpt-5.4-nano-2026-03-17\",\"moderation\":null,\"output\":[],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":\"workflow:instrumentation:01KZ3E9TWTAEK4P4P3F63HKAKC::default::ma\",\"prompt_cache_retention\":\"24h\",\"reasoning\":{\"context\":\"current_turn\",\"effort\":\"low\",\"mode\":\"standard\",\"summary\":\"detailed\"},\"safety_identifier\":null,\"service_tier\":\"auto\",\"store\":false,\"temperature\":1,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tool_usage\":{\"image_gen\":{\"input_tokens\":0,\"input_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"output_tokens\":0,\"output_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"total_tokens\":0},\"web_search\":{\"num_requests\":0}},\"tools\":[{\"type\":\"function\",\"description\":\"Read a file or list a directory. For files, output is truncated to 2000 lines or 50KB — use offset/limit for large files. For directories, returns the list of entries.\",\"name\":\"read\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to read\"},\"offset\":{\"type\":\"number\",\"description\":\"Line number to start from (1-indexed)\"},\"limit\":{\"type\":\"number\",\"description\":\"Maximum number of lines to read\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Write content to a file. Creates the file and parent directories if they do not exist.\",\"name\":\"write\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"content\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to write\"},\"content\":{\"type\":\"string\",\"description\":\"Content to write to the file\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Edit a file using exact text replacement. The oldText must match a unique region of the file. Use replaceAll to replace all occurrences.\",\"name\":\"edit\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"oldText\",\"newText\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to edit\"},\"oldText\":{\"type\":\"string\",\"description\":\"Exact text to find (must be unique)\"},\"newText\":{\"type\":\"string\",\"description\":\"Replacement text\"},\"replaceAll\":{\"type\":\"boolean\",\"description\":\"Replace all occurrences\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Execute a bash command. Returns stdout and stderr. Output is truncated to the last 2000 lines or 50KB.\",\"name\":\"bash\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"command\"],\"properties\":{\"command\":{\"type\":\"string\",\"description\":\"Bash command to execute\"},\"timeout\":{\"type\":\"number\",\"description\":\"Timeout in seconds\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Search file contents for a regex pattern. Returns matching lines with file paths and line numbers.\",\"name\":\"grep\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Search pattern (regex)\"},\"path\":{\"type\":\"string\",\"description\":\"Directory or file to search (default: .)\"},\"include\":{\"type\":\"string\",\"description\":\"Glob filter, e.g. \\\"*.ts\\\"\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Find files by filename pattern using shell find -name semantics. Returns matching file paths.\",\"name\":\"glob\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Filename pattern, e.g. \\\"*.ts\\\"\"},\"path\":{\"type\":\"string\",\"description\":\"Directory to search in (default: .)\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Delegate a focused task to a detached child agent with its own context. Use this for independent research, file exploration, or parallel work. The task returns only its final answer to this conversation. No subagents are currently defined.\",\"name\":\"task\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"prompt\"],\"properties\":{\"description\":{\"type\":\"string\",\"description\":\"Short human-readable label for the delegated work\"},\"prompt\":{\"type\":\"string\",\"description\":\"Focused instructions for the child agent\"},\"agent\":{\"type\":\"string\",\"description\":\"Declared subagent to use for the child agent\"},\"cwd\":{\"type\":\"string\",\"description\":\"Working directory for the child agent. AGENTS.md and skills are discovered from here.\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Return a deterministic lookup result with an id needed by web_search.\",\"name\":\"lookup\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"query\"],\"properties\":{\"query\":{\"type\":\"string\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Search a deterministic local web index. Requires the id returned by lookup.\",\"name\":\"web_search\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"lookupId\",\"query\"],\"properties\":{\"lookupId\":{\"type\":\"string\"},\"query\":{\"type\":\"string\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Summarize the selected deterministic source after web_search returns a URL.\",\"name\":\"summarize_source\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"url\"],\"properties\":{\"url\":{\"type\":\"string\"}}},\"strict\":false}],\"top_logprobs\":0,\"top_p\":0.98,\"truncation\":\"disabled\",\"usage\":null,\"user\":null,\"metadata\":{}},\"sequence_number\":1}", - "event: response.output_item.added\ndata: {\"type\":\"response.output_item.added\",\"item\":{\"id\":\"fc_045458052dc10379016a705b6194148191a2820dc720d327c5\",\"type\":\"function_call\",\"status\":\"in_progress\",\"arguments\":\"\",\"call_id\":\"call_4oiajYhDnktEd6wZwQjSMwpb\",\"name\":\"summarize_source\"},\"output_index\":0,\"sequence_number\":2}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"{\\\"\",\"item_id\":\"fc_045458052dc10379016a705b6194148191a2820dc720d327c5\",\"obfuscation\":\"hK03W2coG6GFdN\",\"output_index\":0,\"sequence_number\":3}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"url\",\"item_id\":\"fc_045458052dc10379016a705b6194148191a2820dc720d327c5\",\"obfuscation\":\"Uu8U6Kh8Nj5Gs\",\"output_index\":0,\"sequence_number\":4}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"\\\":\\\"\",\"item_id\":\"fc_045458052dc10379016a705b6194148191a2820dc720d327c5\",\"obfuscation\":\"sI2ly4ADW4764\",\"output_index\":0,\"sequence_number\":5}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"https\",\"item_id\":\"fc_045458052dc10379016a705b6194148191a2820dc720d327c5\",\"obfuscation\":\"ub8xQfxuyrk\",\"output_index\":0,\"sequence_number\":6}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"://\",\"item_id\":\"fc_045458052dc10379016a705b6194148191a2820dc720d327c5\",\"obfuscation\":\"JvDy4VPF3RT8t\",\"output_index\":0,\"sequence_number\":7}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"example\",\"item_id\":\"fc_045458052dc10379016a705b6194148191a2820dc720d327c5\",\"obfuscation\":\"nUx1jB9C1\",\"output_index\":0,\"sequence_number\":8}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\".test\",\"item_id\":\"fc_045458052dc10379016a705b6194148191a2820dc720d327c5\",\"obfuscation\":\"nESgdh4F5D9\",\"output_index\":0,\"sequence_number\":9}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/fl\",\"item_id\":\"fc_045458052dc10379016a705b6194148191a2820dc720d327c5\",\"obfuscation\":\"3SOSypdEqxfoT\",\"output_index\":0,\"sequence_number\":10}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"ue\",\"item_id\":\"fc_045458052dc10379016a705b6194148191a2820dc720d327c5\",\"obfuscation\":\"WfNnPvGWaIcJYU\",\"output_index\":0,\"sequence_number\":11}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/re\",\"item_id\":\"fc_045458052dc10379016a705b6194148191a2820dc720d327c5\",\"obfuscation\":\"GSIAZIvM9SbTS\",\"output_index\":0,\"sequence_number\":12}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"ason\",\"item_id\":\"fc_045458052dc10379016a705b6194148191a2820dc720d327c5\",\"obfuscation\":\"ELHGgjctvQHZ\",\"output_index\":0,\"sequence_number\":13}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"ing\",\"item_id\":\"fc_045458052dc10379016a705b6194148191a2820dc720d327c5\",\"obfuscation\":\"AGLq1WDbPPiDU\",\"output_index\":0,\"sequence_number\":14}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"-stream\",\"item_id\":\"fc_045458052dc10379016a705b6194148191a2820dc720d327c5\",\"obfuscation\":\"BuKw2YnJj\",\"output_index\":0,\"sequence_number\":15}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"s\",\"item_id\":\"fc_045458052dc10379016a705b6194148191a2820dc720d327c5\",\"obfuscation\":\"szEOAd66hgyQxiR\",\"output_index\":0,\"sequence_number\":16}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"\\\"}\",\"item_id\":\"fc_045458052dc10379016a705b6194148191a2820dc720d327c5\",\"obfuscation\":\"nWuCGD2j8eK0lG\",\"output_index\":0,\"sequence_number\":17}", - "event: response.function_call_arguments.done\ndata: {\"type\":\"response.function_call_arguments.done\",\"arguments\":\"{\\\"url\\\":\\\"https://example.test/flue/reasoning-streams\\\"}\",\"item_id\":\"fc_045458052dc10379016a705b6194148191a2820dc720d327c5\",\"output_index\":0,\"sequence_number\":18}", - "event: response.output_item.done\ndata: {\"type\":\"response.output_item.done\",\"item\":{\"id\":\"fc_045458052dc10379016a705b6194148191a2820dc720d327c5\",\"type\":\"function_call\",\"status\":\"completed\",\"arguments\":\"{\\\"url\\\":\\\"https://example.test/flue/reasoning-streams\\\"}\",\"call_id\":\"call_4oiajYhDnktEd6wZwQjSMwpb\",\"name\":\"summarize_source\"},\"output_index\":0,\"sequence_number\":19}", - "event: response.completed\ndata: {\"type\":\"response.completed\",\"response\":{\"id\":\"resp_045458052dc10379016a705b611e8881919ad2adde8d6874b9\",\"object\":\"response\",\"created_at\":1785748321,\"status\":\"completed\",\"background\":false,\"completed_at\":1785748321,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":null,\"max_tool_calls\":null,\"model\":\"gpt-5.4-nano-2026-03-17\",\"moderation\":null,\"output\":[{\"id\":\"fc_045458052dc10379016a705b6194148191a2820dc720d327c5\",\"type\":\"function_call\",\"status\":\"completed\",\"arguments\":\"{\\\"url\\\":\\\"https://example.test/flue/reasoning-streams\\\"}\",\"call_id\":\"call_4oiajYhDnktEd6wZwQjSMwpb\",\"name\":\"summarize_source\"}],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":\"workflow:instrumentation:01KZ3E9TWTAEK4P4P3F63HKAKC::default::ma\",\"prompt_cache_retention\":\"24h\",\"reasoning\":{\"context\":\"current_turn\",\"effort\":\"low\",\"mode\":\"standard\",\"summary\":\"detailed\"},\"safety_identifier\":null,\"service_tier\":\"default\",\"store\":false,\"temperature\":1,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tool_usage\":{\"image_gen\":{\"input_tokens\":0,\"input_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"output_tokens\":0,\"output_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"total_tokens\":0},\"web_search\":{\"num_requests\":0}},\"tools\":[{\"type\":\"function\",\"description\":\"Read a file or list a directory. For files, output is truncated to 2000 lines or 50KB — use offset/limit for large files. For directories, returns the list of entries.\",\"name\":\"read\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to read\"},\"offset\":{\"type\":\"number\",\"description\":\"Line number to start from (1-indexed)\"},\"limit\":{\"type\":\"number\",\"description\":\"Maximum number of lines to read\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Write content to a file. Creates the file and parent directories if they do not exist.\",\"name\":\"write\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"content\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to write\"},\"content\":{\"type\":\"string\",\"description\":\"Content to write to the file\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Edit a file using exact text replacement. The oldText must match a unique region of the file. Use replaceAll to replace all occurrences.\",\"name\":\"edit\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"oldText\",\"newText\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to edit\"},\"oldText\":{\"type\":\"string\",\"description\":\"Exact text to find (must be unique)\"},\"newText\":{\"type\":\"string\",\"description\":\"Replacement text\"},\"replaceAll\":{\"type\":\"boolean\",\"description\":\"Replace all occurrences\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Execute a bash command. Returns stdout and stderr. Output is truncated to the last 2000 lines or 50KB.\",\"name\":\"bash\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"command\"],\"properties\":{\"command\":{\"type\":\"string\",\"description\":\"Bash command to execute\"},\"timeout\":{\"type\":\"number\",\"description\":\"Timeout in seconds\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Search file contents for a regex pattern. Returns matching lines with file paths and line numbers.\",\"name\":\"grep\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Search pattern (regex)\"},\"path\":{\"type\":\"string\",\"description\":\"Directory or file to search (default: .)\"},\"include\":{\"type\":\"string\",\"description\":\"Glob filter, e.g. \\\"*.ts\\\"\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Find files by filename pattern using shell find -name semantics. Returns matching file paths.\",\"name\":\"glob\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Filename pattern, e.g. \\\"*.ts\\\"\"},\"path\":{\"type\":\"string\",\"description\":\"Directory to search in (default: .)\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Delegate a focused task to a detached child agent with its own context. Use this for independent research, file exploration, or parallel work. The task returns only its final answer to this conversation. No subagents are currently defined.\",\"name\":\"task\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"prompt\"],\"properties\":{\"description\":{\"type\":\"string\",\"description\":\"Short human-readable label for the delegated work\"},\"prompt\":{\"type\":\"string\",\"description\":\"Focused instructions for the child agent\"},\"agent\":{\"type\":\"string\",\"description\":\"Declared subagent to use for the child agent\"},\"cwd\":{\"type\":\"string\",\"description\":\"Working directory for the child agent. AGENTS.md and skills are discovered from here.\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Return a deterministic lookup result with an id needed by web_search.\",\"name\":\"lookup\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"query\"],\"properties\":{\"query\":{\"type\":\"string\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Search a deterministic local web index. Requires the id returned by lookup.\",\"name\":\"web_search\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"lookupId\",\"query\"],\"properties\":{\"lookupId\":{\"type\":\"string\"},\"query\":{\"type\":\"string\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Summarize the selected deterministic source after web_search returns a URL.\",\"name\":\"summarize_source\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"url\"],\"properties\":{\"url\":{\"type\":\"string\"}}},\"strict\":false}],\"top_logprobs\":0,\"top_p\":0.98,\"truncation\":\"disabled\",\"usage\":{\"input_tokens\":1147,\"input_tokens_details\":{\"cache_write_tokens\":0,\"cached_tokens\":0},\"output_tokens\":30,\"output_tokens_details\":{\"reasoning_tokens\":0},\"total_tokens\":1177},\"user\":null,\"metadata\":{}},\"sequence_number\":20}" - ], - "kind": "sse" - }, - "headers": { - "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", - "alt-svc": "h3=\":443\"; ma=86400", - "cf-cache-status": "DYNAMIC", - "cf-ray": "a25432be8c937bb6-IAD", - "connection": "keep-alive", - "content-type": "text/event-stream; charset=utf-8", - "date": "Mon, 03 Aug 2026 09:12:01 GMT", - "openai-organization": "braintrust-data", - "openai-processing-ms": "326", - "openai-project": "proj_vsCSXafhhByzWOThMrJcZiw9", - "openai-version": "2020-10-01", - "server": "cloudflare", - "set-cookie": "[REDACTED]", - "strict-transport-security": "max-age=31536000; includeSubDomains; preload", - "transfer-encoding": "chunked", - "x-content-type-options": "nosniff", - "x-ratelimit-limit-requests": "30000", - "x-ratelimit-limit-tokens": "180000000", - "x-ratelimit-remaining-requests": "29999", - "x-ratelimit-remaining-tokens": "179998398", - "x-ratelimit-reset-requests": "2ms", - "x-ratelimit-reset-tokens": "0s", - "x-request-id": "req_65060d10d7dc405b99f44cac4ba179c7" - }, - "status": 200, - "statusText": "OK" - } - }, - { - "callIndex": 3, - "id": "83cbbf6da482c905", - "matchKey": "POST api.openai.com/v1/responses", - "recordedAt": "2026-08-03T09:12:03.092Z", - "request": { - "body": { - "kind": "json", - "value": { - "include": ["reasoning.encrypted_content"], - "input": [ - { - "content": "You are running in headless mode with no human operator. Work autonomously — never ask questions, never wait for user input. Make your best judgment and proceed independently.\n\nYou are a deterministic Flue instrumentation test agent. Follow user instructions exactly. When asked for a marker, output only that marker and no extra text. When running a local skill file, read it yourself and do not delegate it to a task.\n\n## Available Skills\n\nThe following skills provide specialized instructions for specific tasks. When a task matches a skill description, activate that skill before proceeding so its full instructions are loaded. Skill instructions and supporting resources stay lazy until activation or explicit file reads.\n\n- **e2e-flue-skill** — Deterministic skill used by the Braintrust Flue instrumentation e2e test.\n\nDate: Mon, Aug 3, 2026\nWorking directory: .\n\nDirectory structure:\n.agents\n.flue\n.flue-build\n__cassettes__\n__snapshots__\nassertions.ts\ncassette-filter.mjs\nconstants.mjs\ndist\nnode_modules\npackage.json\npnpm-lock.yaml\nscenario.cli.mjs\nscenario.impl.mjs\nscenario.mjs\nscenario.test.ts\nscenario.ts\nscenario.v2.ts\nv2-assertions.ts\nversions", - "role": "developer" - }, - { - "content": [ - { - "text": "Complete this instrumented research flow. Call exactly one tool per turn and wait for each tool result before choosing the next tool. Step 1: call lookup with query \"flue instrumentation\". Step 2: use the lookup result id as lookupId and call web_search with query \"Braintrust Flue reasoning stream instrumentation\". Step 3: use the first web_search result url and call summarize_source. After summarize_source returns, reply with exactly PROMPT_DONE and no other text.", - "type": "input_text" - } - ], - "role": "user" - }, - { - "arguments": "{\"query\":\"flue instrumentation\"}", - "call_id": "call_rBgonKxWHwhN9E8aQ4VobIOe", - "id": "fc_045458052dc10379016a705b6008408191b27653755cf5bd03", - "name": "lookup", - "type": "function_call" - }, - { - "call_id": "call_rBgonKxWHwhN9E8aQ4VobIOe", - "output": "{\"id\":\"flue-session-2026\",\"query\":\"flue instrumentation\",\"topic\":\"session instrumentation\"}", - "type": "function_call_output" - }, - { - "arguments": "{\"lookupId\":\"flue-session-2026\",\"query\":\"Braintrust Flue reasoning stream instrumentation\"}", - "call_id": "call_fGesVCba2aABMoxSNfVnDVnR", - "id": "fc_045458052dc10379016a705b60c92c81918da2fab9710ee199", - "name": "web_search", - "type": "function_call" - }, - { - "call_id": "call_fGesVCba2aABMoxSNfVnDVnR", - "output": "{\"lookupId\":\"flue-session-2026\",\"query\":\"Braintrust Flue reasoning stream instrumentation\",\"results\":[{\"title\":\"Flue reasoning stream instrumentation\",\"url\":\"https://example.test/flue/reasoning-streams\"}]}", - "type": "function_call_output" - }, - { - "arguments": "{\"url\":\"https://example.test/flue/reasoning-streams\"}", - "call_id": "call_4oiajYhDnktEd6wZwQjSMwpb", - "id": "fc_045458052dc10379016a705b6194148191a2820dc720d327c5", - "name": "summarize_source", - "type": "function_call" - }, - { - "call_id": "call_4oiajYhDnktEd6wZwQjSMwpb", - "output": "{\"summary\":\"Flue emits reasoning, tool execution, and LLM turn events separately.\",\"url\":\"https://example.test/flue/reasoning-streams\"}", - "type": "function_call_output" - } - ], - "model": "gpt-5.4-nano", - "prompt_cache_key": "[REDACTED]", - "reasoning": { - "effort": "low", - "summary": "auto" - }, - "store": false, - "stream": true, - "tools": [ - { - "description": "Read a file or list a directory. For files, output is truncated to 2000 lines or 50KB — use offset/limit for large files. For directories, returns the list of entries.", - "name": "read", - "parameters": { - "properties": { - "limit": { - "description": "Maximum number of lines to read", - "type": "number" - }, - "offset": { - "description": "Line number to start from (1-indexed)", - "type": "number" - }, - "path": { - "description": "Path to the file to read", - "type": "string" - } - }, - "required": ["path"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Write content to a file. Creates the file and parent directories if they do not exist.", - "name": "write", - "parameters": { - "properties": { - "content": { - "description": "Content to write to the file", - "type": "string" - }, - "path": { - "description": "Path to the file to write", - "type": "string" - } - }, - "required": ["path", "content"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Edit a file using exact text replacement. The oldText must match a unique region of the file. Use replaceAll to replace all occurrences.", - "name": "edit", - "parameters": { - "properties": { - "newText": { - "description": "Replacement text", - "type": "string" - }, - "oldText": { - "description": "Exact text to find (must be unique)", - "type": "string" - }, - "path": { - "description": "Path to the file to edit", - "type": "string" - }, - "replaceAll": { - "description": "Replace all occurrences", - "type": "boolean" - } - }, - "required": ["path", "oldText", "newText"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Execute a bash command. Returns stdout and stderr. Output is truncated to the last 2000 lines or 50KB.", - "name": "bash", - "parameters": { - "properties": { - "command": { - "description": "Bash command to execute", - "type": "string" - }, - "timeout": { - "description": "Timeout in seconds", - "type": "number" - } - }, - "required": ["command"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Search file contents for a regex pattern. Returns matching lines with file paths and line numbers.", - "name": "grep", - "parameters": { - "properties": { - "include": { - "description": "Glob filter, e.g. \"*.ts\"", - "type": "string" - }, - "path": { - "description": "Directory or file to search (default: .)", - "type": "string" - }, - "pattern": { - "description": "Search pattern (regex)", - "type": "string" - } - }, - "required": ["pattern"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Find files by filename pattern using shell find -name semantics. Returns matching file paths.", - "name": "glob", - "parameters": { - "properties": { - "path": { - "description": "Directory to search in (default: .)", - "type": "string" - }, - "pattern": { - "description": "Filename pattern, e.g. \"*.ts\"", - "type": "string" - } - }, - "required": ["pattern"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Delegate a focused task to a detached child agent with its own context. Use this for independent research, file exploration, or parallel work. The task returns only its final answer to this conversation. No subagents are currently defined.", - "name": "task", - "parameters": { - "properties": { - "agent": { - "description": "Declared subagent to use for the child agent", - "type": "string" - }, - "cwd": { - "description": "Working directory for the child agent. AGENTS.md and skills are discovered from here.", - "type": "string" - }, - "description": { - "description": "Short human-readable label for the delegated work", - "type": "string" - }, - "prompt": { - "description": "Focused instructions for the child agent", - "type": "string" - } - }, - "required": ["prompt"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Return a deterministic lookup result with an id needed by web_search.", - "name": "lookup", - "parameters": { - "properties": { - "query": { - "type": "string" - } - }, - "required": ["query"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Search a deterministic local web index. Requires the id returned by lookup.", - "name": "web_search", - "parameters": { - "properties": { - "lookupId": { - "type": "string" - }, - "query": { - "type": "string" - } - }, - "required": ["lookupId", "query"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Summarize the selected deterministic source after web_search returns a URL.", - "name": "summarize_source", - "parameters": { - "properties": { - "url": { - "type": "string" - } - }, - "required": ["url"], - "type": "object" - }, - "strict": false, - "type": "function" - } - ] - } - }, - "headers": {}, - "method": "POST", - "url": "https://api.openai.com/v1/responses" - }, - "response": { - "body": { - "chunks": [ - "event: response.created\ndata: {\"type\":\"response.created\",\"response\":{\"id\":\"resp_045458052dc10379016a705b6286dc8191be79bd3493b23c38\",\"object\":\"response\",\"created_at\":1785748322,\"status\":\"in_progress\",\"background\":false,\"completed_at\":null,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":null,\"max_tool_calls\":null,\"model\":\"gpt-5.4-nano-2026-03-17\",\"moderation\":null,\"output\":[],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":\"workflow:instrumentation:01KZ3E9TWTAEK4P4P3F63HKAKC::default::ma\",\"prompt_cache_retention\":\"24h\",\"reasoning\":{\"context\":\"current_turn\",\"effort\":\"low\",\"mode\":\"standard\",\"summary\":\"detailed\"},\"safety_identifier\":null,\"service_tier\":\"auto\",\"store\":false,\"temperature\":1,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tool_usage\":{\"image_gen\":{\"input_tokens\":0,\"input_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"output_tokens\":0,\"output_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"total_tokens\":0},\"web_search\":{\"num_requests\":0}},\"tools\":[{\"type\":\"function\",\"description\":\"Read a file or list a directory. For files, output is truncated to 2000 lines or 50KB — use offset/limit for large files. For directories, returns the list of entries.\",\"name\":\"read\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to read\"},\"offset\":{\"type\":\"number\",\"description\":\"Line number to start from (1-indexed)\"},\"limit\":{\"type\":\"number\",\"description\":\"Maximum number of lines to read\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Write content to a file. Creates the file and parent directories if they do not exist.\",\"name\":\"write\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"content\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to write\"},\"content\":{\"type\":\"string\",\"description\":\"Content to write to the file\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Edit a file using exact text replacement. The oldText must match a unique region of the file. Use replaceAll to replace all occurrences.\",\"name\":\"edit\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"oldText\",\"newText\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to edit\"},\"oldText\":{\"type\":\"string\",\"description\":\"Exact text to find (must be unique)\"},\"newText\":{\"type\":\"string\",\"description\":\"Replacement text\"},\"replaceAll\":{\"type\":\"boolean\",\"description\":\"Replace all occurrences\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Execute a bash command. Returns stdout and stderr. Output is truncated to the last 2000 lines or 50KB.\",\"name\":\"bash\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"command\"],\"properties\":{\"command\":{\"type\":\"string\",\"description\":\"Bash command to execute\"},\"timeout\":{\"type\":\"number\",\"description\":\"Timeout in seconds\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Search file contents for a regex pattern. Returns matching lines with file paths and line numbers.\",\"name\":\"grep\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Search pattern (regex)\"},\"path\":{\"type\":\"string\",\"description\":\"Directory or file to search (default: .)\"},\"include\":{\"type\":\"string\",\"description\":\"Glob filter, e.g. \\\"*.ts\\\"\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Find files by filename pattern using shell find -name semantics. Returns matching file paths.\",\"name\":\"glob\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Filename pattern, e.g. \\\"*.ts\\\"\"},\"path\":{\"type\":\"string\",\"description\":\"Directory to search in (default: .)\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Delegate a focused task to a detached child agent with its own context. Use this for independent research, file exploration, or parallel work. The task returns only its final answer to this conversation. No subagents are currently defined.\",\"name\":\"task\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"prompt\"],\"properties\":{\"description\":{\"type\":\"string\",\"description\":\"Short human-readable label for the delegated work\"},\"prompt\":{\"type\":\"string\",\"description\":\"Focused instructions for the child agent\"},\"agent\":{\"type\":\"string\",\"description\":\"Declared subagent to use for the child agent\"},\"cwd\":{\"type\":\"string\",\"description\":\"Working directory for the child agent. AGENTS.md and skills are discovered from here.\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Return a deterministic lookup result with an id needed by web_search.\",\"name\":\"lookup\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"query\"],\"properties\":{\"query\":{\"type\":\"string\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Search a deterministic local web index. Requires the id returned by lookup.\",\"name\":\"web_search\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"lookupId\",\"query\"],\"properties\":{\"lookupId\":{\"type\":\"string\"},\"query\":{\"type\":\"string\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Summarize the selected deterministic source after web_search returns a URL.\",\"name\":\"summarize_source\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"url\"],\"properties\":{\"url\":{\"type\":\"string\"}}},\"strict\":false}],\"top_logprobs\":0,\"top_p\":0.98,\"truncation\":\"disabled\",\"usage\":null,\"user\":null,\"metadata\":{}},\"sequence_number\":0}", - "event: response.in_progress\ndata: {\"type\":\"response.in_progress\",\"response\":{\"id\":\"resp_045458052dc10379016a705b6286dc8191be79bd3493b23c38\",\"object\":\"response\",\"created_at\":1785748322,\"status\":\"in_progress\",\"background\":false,\"completed_at\":null,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":null,\"max_tool_calls\":null,\"model\":\"gpt-5.4-nano-2026-03-17\",\"moderation\":null,\"output\":[],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":\"workflow:instrumentation:01KZ3E9TWTAEK4P4P3F63HKAKC::default::ma\",\"prompt_cache_retention\":\"24h\",\"reasoning\":{\"context\":\"current_turn\",\"effort\":\"low\",\"mode\":\"standard\",\"summary\":\"detailed\"},\"safety_identifier\":null,\"service_tier\":\"auto\",\"store\":false,\"temperature\":1,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tool_usage\":{\"image_gen\":{\"input_tokens\":0,\"input_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"output_tokens\":0,\"output_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"total_tokens\":0},\"web_search\":{\"num_requests\":0}},\"tools\":[{\"type\":\"function\",\"description\":\"Read a file or list a directory. For files, output is truncated to 2000 lines or 50KB — use offset/limit for large files. For directories, returns the list of entries.\",\"name\":\"read\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to read\"},\"offset\":{\"type\":\"number\",\"description\":\"Line number to start from (1-indexed)\"},\"limit\":{\"type\":\"number\",\"description\":\"Maximum number of lines to read\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Write content to a file. Creates the file and parent directories if they do not exist.\",\"name\":\"write\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"content\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to write\"},\"content\":{\"type\":\"string\",\"description\":\"Content to write to the file\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Edit a file using exact text replacement. The oldText must match a unique region of the file. Use replaceAll to replace all occurrences.\",\"name\":\"edit\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"oldText\",\"newText\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to edit\"},\"oldText\":{\"type\":\"string\",\"description\":\"Exact text to find (must be unique)\"},\"newText\":{\"type\":\"string\",\"description\":\"Replacement text\"},\"replaceAll\":{\"type\":\"boolean\",\"description\":\"Replace all occurrences\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Execute a bash command. Returns stdout and stderr. Output is truncated to the last 2000 lines or 50KB.\",\"name\":\"bash\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"command\"],\"properties\":{\"command\":{\"type\":\"string\",\"description\":\"Bash command to execute\"},\"timeout\":{\"type\":\"number\",\"description\":\"Timeout in seconds\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Search file contents for a regex pattern. Returns matching lines with file paths and line numbers.\",\"name\":\"grep\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Search pattern (regex)\"},\"path\":{\"type\":\"string\",\"description\":\"Directory or file to search (default: .)\"},\"include\":{\"type\":\"string\",\"description\":\"Glob filter, e.g. \\\"*.ts\\\"\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Find files by filename pattern using shell find -name semantics. Returns matching file paths.\",\"name\":\"glob\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Filename pattern, e.g. \\\"*.ts\\\"\"},\"path\":{\"type\":\"string\",\"description\":\"Directory to search in (default: .)\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Delegate a focused task to a detached child agent with its own context. Use this for independent research, file exploration, or parallel work. The task returns only its final answer to this conversation. No subagents are currently defined.\",\"name\":\"task\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"prompt\"],\"properties\":{\"description\":{\"type\":\"string\",\"description\":\"Short human-readable label for the delegated work\"},\"prompt\":{\"type\":\"string\",\"description\":\"Focused instructions for the child agent\"},\"agent\":{\"type\":\"string\",\"description\":\"Declared subagent to use for the child agent\"},\"cwd\":{\"type\":\"string\",\"description\":\"Working directory for the child agent. AGENTS.md and skills are discovered from here.\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Return a deterministic lookup result with an id needed by web_search.\",\"name\":\"lookup\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"query\"],\"properties\":{\"query\":{\"type\":\"string\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Search a deterministic local web index. Requires the id returned by lookup.\",\"name\":\"web_search\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"lookupId\",\"query\"],\"properties\":{\"lookupId\":{\"type\":\"string\"},\"query\":{\"type\":\"string\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Summarize the selected deterministic source after web_search returns a URL.\",\"name\":\"summarize_source\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"url\"],\"properties\":{\"url\":{\"type\":\"string\"}}},\"strict\":false}],\"top_logprobs\":0,\"top_p\":0.98,\"truncation\":\"disabled\",\"usage\":null,\"user\":null,\"metadata\":{}},\"sequence_number\":1}", - "event: response.output_item.added\ndata: {\"type\":\"response.output_item.added\",\"item\":{\"id\":\"msg_045458052dc10379016a705b62dbd8819189b8287061debf12\",\"type\":\"message\",\"status\":\"in_progress\",\"content\":[],\"phase\":\"final_answer\",\"role\":\"assistant\"},\"output_index\":0,\"sequence_number\":2}", - "event: response.content_part.added\ndata: {\"type\":\"response.content_part.added\",\"content_index\":0,\"item_id\":\"msg_045458052dc10379016a705b62dbd8819189b8287061debf12\",\"output_index\":0,\"part\":{\"type\":\"output_text\",\"annotations\":[],\"logprobs\":[],\"text\":\"\"},\"sequence_number\":3}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"PROM\",\"item_id\":\"msg_045458052dc10379016a705b62dbd8819189b8287061debf12\",\"logprobs\":[],\"obfuscation\":\"ttkbAlDp50d6\",\"output_index\":0,\"sequence_number\":4}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"PT\",\"item_id\":\"msg_045458052dc10379016a705b62dbd8819189b8287061debf12\",\"logprobs\":[],\"obfuscation\":\"Z6PXaim62hCyKh\",\"output_index\":0,\"sequence_number\":5}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"_DONE\",\"item_id\":\"msg_045458052dc10379016a705b62dbd8819189b8287061debf12\",\"logprobs\":[],\"obfuscation\":\"MiAXFIn3IuM\",\"output_index\":0,\"sequence_number\":6}", - "event: response.output_text.done\ndata: {\"type\":\"response.output_text.done\",\"content_index\":0,\"item_id\":\"msg_045458052dc10379016a705b62dbd8819189b8287061debf12\",\"logprobs\":[],\"output_index\":0,\"sequence_number\":7,\"text\":\"PROMPT_DONE\"}", - "event: response.content_part.done\ndata: {\"type\":\"response.content_part.done\",\"content_index\":0,\"item_id\":\"msg_045458052dc10379016a705b62dbd8819189b8287061debf12\",\"output_index\":0,\"part\":{\"type\":\"output_text\",\"annotations\":[],\"logprobs\":[],\"text\":\"PROMPT_DONE\"},\"sequence_number\":8}", - "event: response.output_item.done\ndata: {\"type\":\"response.output_item.done\",\"item\":{\"id\":\"msg_045458052dc10379016a705b62dbd8819189b8287061debf12\",\"type\":\"message\",\"status\":\"completed\",\"content\":[{\"type\":\"output_text\",\"annotations\":[],\"logprobs\":[],\"text\":\"PROMPT_DONE\"}],\"phase\":\"final_answer\",\"role\":\"assistant\"},\"output_index\":0,\"sequence_number\":9}", - "event: response.completed\ndata: {\"type\":\"response.completed\",\"response\":{\"id\":\"resp_045458052dc10379016a705b6286dc8191be79bd3493b23c38\",\"object\":\"response\",\"created_at\":1785748322,\"status\":\"completed\",\"background\":false,\"completed_at\":1785748323,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":null,\"max_tool_calls\":null,\"model\":\"gpt-5.4-nano-2026-03-17\",\"moderation\":null,\"output\":[{\"id\":\"msg_045458052dc10379016a705b62dbd8819189b8287061debf12\",\"type\":\"message\",\"status\":\"completed\",\"content\":[{\"type\":\"output_text\",\"annotations\":[],\"logprobs\":[],\"text\":\"PROMPT_DONE\"}],\"phase\":\"final_answer\",\"role\":\"assistant\"}],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":\"workflow:instrumentation:01KZ3E9TWTAEK4P4P3F63HKAKC::default::ma\",\"prompt_cache_retention\":\"24h\",\"reasoning\":{\"context\":\"current_turn\",\"effort\":\"low\",\"mode\":\"standard\",\"summary\":\"detailed\"},\"safety_identifier\":null,\"service_tier\":\"default\",\"store\":false,\"temperature\":1,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tool_usage\":{\"image_gen\":{\"input_tokens\":0,\"input_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"output_tokens\":0,\"output_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"total_tokens\":0},\"web_search\":{\"num_requests\":0}},\"tools\":[{\"type\":\"function\",\"description\":\"Read a file or list a directory. For files, output is truncated to 2000 lines or 50KB — use offset/limit for large files. For directories, returns the list of entries.\",\"name\":\"read\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to read\"},\"offset\":{\"type\":\"number\",\"description\":\"Line number to start from (1-indexed)\"},\"limit\":{\"type\":\"number\",\"description\":\"Maximum number of lines to read\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Write content to a file. Creates the file and parent directories if they do not exist.\",\"name\":\"write\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"content\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to write\"},\"content\":{\"type\":\"string\",\"description\":\"Content to write to the file\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Edit a file using exact text replacement. The oldText must match a unique region of the file. Use replaceAll to replace all occurrences.\",\"name\":\"edit\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"oldText\",\"newText\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to edit\"},\"oldText\":{\"type\":\"string\",\"description\":\"Exact text to find (must be unique)\"},\"newText\":{\"type\":\"string\",\"description\":\"Replacement text\"},\"replaceAll\":{\"type\":\"boolean\",\"description\":\"Replace all occurrences\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Execute a bash command. Returns stdout and stderr. Output is truncated to the last 2000 lines or 50KB.\",\"name\":\"bash\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"command\"],\"properties\":{\"command\":{\"type\":\"string\",\"description\":\"Bash command to execute\"},\"timeout\":{\"type\":\"number\",\"description\":\"Timeout in seconds\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Search file contents for a regex pattern. Returns matching lines with file paths and line numbers.\",\"name\":\"grep\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Search pattern (regex)\"},\"path\":{\"type\":\"string\",\"description\":\"Directory or file to search (default: .)\"},\"include\":{\"type\":\"string\",\"description\":\"Glob filter, e.g. \\\"*.ts\\\"\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Find files by filename pattern using shell find -name semantics. Returns matching file paths.\",\"name\":\"glob\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Filename pattern, e.g. \\\"*.ts\\\"\"},\"path\":{\"type\":\"string\",\"description\":\"Directory to search in (default: .)\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Delegate a focused task to a detached child agent with its own context. Use this for independent research, file exploration, or parallel work. The task returns only its final answer to this conversation. No subagents are currently defined.\",\"name\":\"task\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"prompt\"],\"properties\":{\"description\":{\"type\":\"string\",\"description\":\"Short human-readable label for the delegated work\"},\"prompt\":{\"type\":\"string\",\"description\":\"Focused instructions for the child agent\"},\"agent\":{\"type\":\"string\",\"description\":\"Declared subagent to use for the child agent\"},\"cwd\":{\"type\":\"string\",\"description\":\"Working directory for the child agent. AGENTS.md and skills are discovered from here.\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Return a deterministic lookup result with an id needed by web_search.\",\"name\":\"lookup\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"query\"],\"properties\":{\"query\":{\"type\":\"string\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Search a deterministic local web index. Requires the id returned by lookup.\",\"name\":\"web_search\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"lookupId\",\"query\"],\"properties\":{\"lookupId\":{\"type\":\"string\"},\"query\":{\"type\":\"string\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Summarize the selected deterministic source after web_search returns a URL.\",\"name\":\"summarize_source\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"url\"],\"properties\":{\"url\":{\"type\":\"string\"}}},\"strict\":false}],\"top_logprobs\":0,\"top_p\":0.98,\"truncation\":\"disabled\",\"usage\":{\"input_tokens\":1222,\"input_tokens_details\":{\"cache_write_tokens\":0,\"cached_tokens\":0},\"output_tokens\":7,\"output_tokens_details\":{\"reasoning_tokens\":0},\"total_tokens\":1229},\"user\":null,\"metadata\":{}},\"sequence_number\":10}" - ], - "kind": "sse" - }, - "headers": { - "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", - "alt-svc": "h3=\":443\"; ma=86400", - "cf-cache-status": "DYNAMIC", - "cf-ray": "a25432c7682f7bb6-IAD", - "connection": "keep-alive", - "content-type": "text/event-stream; charset=utf-8", - "date": "Mon, 03 Aug 2026 09:12:02 GMT", - "openai-organization": "braintrust-data", - "openai-processing-ms": "281", - "openai-project": "proj_vsCSXafhhByzWOThMrJcZiw9", - "openai-version": "2020-10-01", - "server": "cloudflare", - "set-cookie": "[REDACTED]", - "strict-transport-security": "max-age=31536000; includeSubDomains; preload", - "transfer-encoding": "chunked", - "x-content-type-options": "nosniff", - "x-ratelimit-limit-requests": "30000", - "x-ratelimit-limit-tokens": "180000000", - "x-ratelimit-remaining-requests": "29999", - "x-ratelimit-remaining-tokens": "179998323", - "x-ratelimit-reset-requests": "2ms", - "x-ratelimit-reset-tokens": "0s", - "x-request-id": "req_89fa79409a904f43a453a36d2ca890d0" - }, - "status": 200, - "statusText": "OK" - } - }, - { - "callIndex": 4, - "id": "417e64addcf59da7", - "matchKey": "POST api.openai.com/v1/responses", - "recordedAt": "2026-08-03T09:12:04.374Z", - "request": { - "body": { - "kind": "json", - "value": { - "input": [ - { - "content": "You are running in headless mode with no human operator. Work autonomously — never ask questions, never wait for user input. Make your best judgment and proceed independently.\n\nYou are a deterministic Flue instrumentation test agent. Follow user instructions exactly. When asked for a marker, output only that marker and no extra text. When running a local skill file, read it yourself and do not delegate it to a task.\n\n## Available Skills\n\nThe following skills provide specialized instructions for specific tasks. When a task matches a skill description, activate that skill before proceeding so its full instructions are loaded. Skill instructions and supporting resources stay lazy until activation or explicit file reads.\n\n- **e2e-flue-skill** — Deterministic skill used by the Braintrust Flue instrumentation e2e test.\n\nDate: Mon, Aug 3, 2026\nWorking directory: .\n\nDirectory structure:\n.agents\n.flue\n.flue-build\n__cassettes__\n__snapshots__\nassertions.ts\ncassette-filter.mjs\nconstants.mjs\ndist\nnode_modules\npackage.json\npnpm-lock.yaml\nscenario.cli.mjs\nscenario.impl.mjs\nscenario.mjs\nscenario.test.ts\nscenario.ts\nscenario.v2.ts\nv2-assertions.ts\nversions", - "role": "developer" - }, - { - "content": [ - { - "text": "Run the skill named \"e2e-flue-skill\".\n\nArguments:\n{\n \"marker\": \"SKILL_DONE\"\n}", - "type": "input_text" - } - ], - "role": "user" - } - ], - "model": "gpt-5.4-nano", - "prompt_cache_key": "[REDACTED]", - "reasoning": { - "effort": "none" - }, - "store": false, - "stream": true, - "tools": [ - { - "description": "Read a file or list a directory. For files, output is truncated to 2000 lines or 50KB — use offset/limit for large files. For directories, returns the list of entries.", - "name": "read", - "parameters": { - "properties": { - "limit": { - "description": "Maximum number of lines to read", - "type": "number" - }, - "offset": { - "description": "Line number to start from (1-indexed)", - "type": "number" - }, - "path": { - "description": "Path to the file to read", - "type": "string" - } - }, - "required": ["path"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Write content to a file. Creates the file and parent directories if they do not exist.", - "name": "write", - "parameters": { - "properties": { - "content": { - "description": "Content to write to the file", - "type": "string" - }, - "path": { - "description": "Path to the file to write", - "type": "string" - } - }, - "required": ["path", "content"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Edit a file using exact text replacement. The oldText must match a unique region of the file. Use replaceAll to replace all occurrences.", - "name": "edit", - "parameters": { - "properties": { - "newText": { - "description": "Replacement text", - "type": "string" - }, - "oldText": { - "description": "Exact text to find (must be unique)", - "type": "string" - }, - "path": { - "description": "Path to the file to edit", - "type": "string" - }, - "replaceAll": { - "description": "Replace all occurrences", - "type": "boolean" - } - }, - "required": ["path", "oldText", "newText"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Execute a bash command. Returns stdout and stderr. Output is truncated to the last 2000 lines or 50KB.", - "name": "bash", - "parameters": { - "properties": { - "command": { - "description": "Bash command to execute", - "type": "string" - }, - "timeout": { - "description": "Timeout in seconds", - "type": "number" - } - }, - "required": ["command"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Search file contents for a regex pattern. Returns matching lines with file paths and line numbers.", - "name": "grep", - "parameters": { - "properties": { - "include": { - "description": "Glob filter, e.g. \"*.ts\"", - "type": "string" - }, - "path": { - "description": "Directory or file to search (default: .)", - "type": "string" - }, - "pattern": { - "description": "Search pattern (regex)", - "type": "string" - } - }, - "required": ["pattern"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Find files by filename pattern using shell find -name semantics. Returns matching file paths.", - "name": "glob", - "parameters": { - "properties": { - "path": { - "description": "Directory to search in (default: .)", - "type": "string" - }, - "pattern": { - "description": "Filename pattern, e.g. \"*.ts\"", - "type": "string" - } - }, - "required": ["pattern"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Delegate a focused task to a detached child agent with its own context. Use this for independent research, file exploration, or parallel work. The task returns only its final answer to this conversation. No subagents are currently defined.", - "name": "task", - "parameters": { - "properties": { - "agent": { - "description": "Declared subagent to use for the child agent", - "type": "string" - }, - "cwd": { - "description": "Working directory for the child agent. AGENTS.md and skills are discovered from here.", - "type": "string" - }, - "description": { - "description": "Short human-readable label for the delegated work", - "type": "string" - }, - "prompt": { - "description": "Focused instructions for the child agent", - "type": "string" - } - }, - "required": ["prompt"], - "type": "object" - }, - "strict": false, - "type": "function" - } - ] - } - }, - "headers": {}, - "method": "POST", - "url": "https://api.openai.com/v1/responses" - }, - "response": { - "body": { - "chunks": [ - "event: response.created\ndata: {\"type\":\"response.created\",\"response\":{\"id\":\"resp_0f099b3fce0721fd016a705b63235881a393625d6974243e44\",\"object\":\"response\",\"created_at\":1785748323,\"status\":\"in_progress\",\"background\":false,\"completed_at\":null,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":null,\"max_tool_calls\":null,\"model\":\"gpt-5.4-nano-2026-03-17\",\"moderation\":null,\"output\":[],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":\"workflow:instrumentation:01KZ3E9TWTAEK4P4P3F63HKAKC::default::sk\",\"prompt_cache_retention\":\"24h\",\"reasoning\":{\"context\":\"current_turn\",\"effort\":\"none\",\"mode\":\"standard\",\"summary\":null},\"safety_identifier\":null,\"service_tier\":\"auto\",\"store\":false,\"temperature\":1,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tool_usage\":{\"image_gen\":{\"input_tokens\":0,\"input_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"output_tokens\":0,\"output_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"total_tokens\":0},\"web_search\":{\"num_requests\":0}},\"tools\":[{\"type\":\"function\",\"description\":\"Read a file or list a directory. For files, output is truncated to 2000 lines or 50KB — use offset/limit for large files. For directories, returns the list of entries.\",\"name\":\"read\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to read\"},\"offset\":{\"type\":\"number\",\"description\":\"Line number to start from (1-indexed)\"},\"limit\":{\"type\":\"number\",\"description\":\"Maximum number of lines to read\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Write content to a file. Creates the file and parent directories if they do not exist.\",\"name\":\"write\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"content\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to write\"},\"content\":{\"type\":\"string\",\"description\":\"Content to write to the file\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Edit a file using exact text replacement. The oldText must match a unique region of the file. Use replaceAll to replace all occurrences.\",\"name\":\"edit\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"oldText\",\"newText\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to edit\"},\"oldText\":{\"type\":\"string\",\"description\":\"Exact text to find (must be unique)\"},\"newText\":{\"type\":\"string\",\"description\":\"Replacement text\"},\"replaceAll\":{\"type\":\"boolean\",\"description\":\"Replace all occurrences\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Execute a bash command. Returns stdout and stderr. Output is truncated to the last 2000 lines or 50KB.\",\"name\":\"bash\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"command\"],\"properties\":{\"command\":{\"type\":\"string\",\"description\":\"Bash command to execute\"},\"timeout\":{\"type\":\"number\",\"description\":\"Timeout in seconds\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Search file contents for a regex pattern. Returns matching lines with file paths and line numbers.\",\"name\":\"grep\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Search pattern (regex)\"},\"path\":{\"type\":\"string\",\"description\":\"Directory or file to search (default: .)\"},\"include\":{\"type\":\"string\",\"description\":\"Glob filter, e.g. \\\"*.ts\\\"\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Find files by filename pattern using shell find -name semantics. Returns matching file paths.\",\"name\":\"glob\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Filename pattern, e.g. \\\"*.ts\\\"\"},\"path\":{\"type\":\"string\",\"description\":\"Directory to search in (default: .)\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Delegate a focused task to a detached child agent with its own context. Use this for independent research, file exploration, or parallel work. The task returns only its final answer to this conversation. No subagents are currently defined.\",\"name\":\"task\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"prompt\"],\"properties\":{\"description\":{\"type\":\"string\",\"description\":\"Short human-readable label for the delegated work\"},\"prompt\":{\"type\":\"string\",\"description\":\"Focused instructions for the child agent\"},\"agent\":{\"type\":\"string\",\"description\":\"Declared subagent to use for the child agent\"},\"cwd\":{\"type\":\"string\",\"description\":\"Working directory for the child agent. AGENTS.md and skills are discovered from here.\"}}},\"strict\":false}],\"top_logprobs\":0,\"top_p\":0.98,\"truncation\":\"disabled\",\"usage\":null,\"user\":null,\"metadata\":{}},\"sequence_number\":0}", - "event: response.in_progress\ndata: {\"type\":\"response.in_progress\",\"response\":{\"id\":\"resp_0f099b3fce0721fd016a705b63235881a393625d6974243e44\",\"object\":\"response\",\"created_at\":1785748323,\"status\":\"in_progress\",\"background\":false,\"completed_at\":null,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":null,\"max_tool_calls\":null,\"model\":\"gpt-5.4-nano-2026-03-17\",\"moderation\":null,\"output\":[],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":\"workflow:instrumentation:01KZ3E9TWTAEK4P4P3F63HKAKC::default::sk\",\"prompt_cache_retention\":\"24h\",\"reasoning\":{\"context\":\"current_turn\",\"effort\":\"none\",\"mode\":\"standard\",\"summary\":null},\"safety_identifier\":null,\"service_tier\":\"auto\",\"store\":false,\"temperature\":1,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tool_usage\":{\"image_gen\":{\"input_tokens\":0,\"input_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"output_tokens\":0,\"output_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"total_tokens\":0},\"web_search\":{\"num_requests\":0}},\"tools\":[{\"type\":\"function\",\"description\":\"Read a file or list a directory. For files, output is truncated to 2000 lines or 50KB — use offset/limit for large files. For directories, returns the list of entries.\",\"name\":\"read\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to read\"},\"offset\":{\"type\":\"number\",\"description\":\"Line number to start from (1-indexed)\"},\"limit\":{\"type\":\"number\",\"description\":\"Maximum number of lines to read\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Write content to a file. Creates the file and parent directories if they do not exist.\",\"name\":\"write\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"content\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to write\"},\"content\":{\"type\":\"string\",\"description\":\"Content to write to the file\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Edit a file using exact text replacement. The oldText must match a unique region of the file. Use replaceAll to replace all occurrences.\",\"name\":\"edit\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"oldText\",\"newText\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to edit\"},\"oldText\":{\"type\":\"string\",\"description\":\"Exact text to find (must be unique)\"},\"newText\":{\"type\":\"string\",\"description\":\"Replacement text\"},\"replaceAll\":{\"type\":\"boolean\",\"description\":\"Replace all occurrences\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Execute a bash command. Returns stdout and stderr. Output is truncated to the last 2000 lines or 50KB.\",\"name\":\"bash\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"command\"],\"properties\":{\"command\":{\"type\":\"string\",\"description\":\"Bash command to execute\"},\"timeout\":{\"type\":\"number\",\"description\":\"Timeout in seconds\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Search file contents for a regex pattern. Returns matching lines with file paths and line numbers.\",\"name\":\"grep\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Search pattern (regex)\"},\"path\":{\"type\":\"string\",\"description\":\"Directory or file to search (default: .)\"},\"include\":{\"type\":\"string\",\"description\":\"Glob filter, e.g. \\\"*.ts\\\"\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Find files by filename pattern using shell find -name semantics. Returns matching file paths.\",\"name\":\"glob\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Filename pattern, e.g. \\\"*.ts\\\"\"},\"path\":{\"type\":\"string\",\"description\":\"Directory to search in (default: .)\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Delegate a focused task to a detached child agent with its own context. Use this for independent research, file exploration, or parallel work. The task returns only its final answer to this conversation. No subagents are currently defined.\",\"name\":\"task\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"prompt\"],\"properties\":{\"description\":{\"type\":\"string\",\"description\":\"Short human-readable label for the delegated work\"},\"prompt\":{\"type\":\"string\",\"description\":\"Focused instructions for the child agent\"},\"agent\":{\"type\":\"string\",\"description\":\"Declared subagent to use for the child agent\"},\"cwd\":{\"type\":\"string\",\"description\":\"Working directory for the child agent. AGENTS.md and skills are discovered from here.\"}}},\"strict\":false}],\"top_logprobs\":0,\"top_p\":0.98,\"truncation\":\"disabled\",\"usage\":null,\"user\":null,\"metadata\":{}},\"sequence_number\":1}", - "event: response.output_item.added\ndata: {\"type\":\"response.output_item.added\",\"item\":{\"id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"type\":\"function_call\",\"status\":\"in_progress\",\"arguments\":\"\",\"call_id\":\"call_R2a5LOml0NZqq5XTcTCKfbMW\",\"name\":\"task\"},\"output_index\":0,\"sequence_number\":2}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"{\\\"\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"CbHdtJmEy68R82\",\"output_index\":0,\"sequence_number\":3}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"description\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"N0n5J\",\"output_index\":0,\"sequence_number\":4}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"\\\":\\\"\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"oVS9emhElTbAe\",\"output_index\":0,\"sequence_number\":5}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"Run\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"44SOZV25Sx36v\",\"output_index\":0,\"sequence_number\":6}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" the\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"5rEAHjD7s9AY\",\"output_index\":0,\"sequence_number\":7}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" deterministic\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"XS\",\"output_index\":0,\"sequence_number\":8}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" Fl\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"M9xvWA7f348PV\",\"output_index\":0,\"sequence_number\":9}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"ue\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"Q6wOJG5PkJoZVd\",\"output_index\":0,\"sequence_number\":10}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" instrumentation\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"\",\"output_index\":0,\"sequence_number\":11}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" e\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"YPWsSbID38RZ8M\",\"output_index\":0,\"sequence_number\":12}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"2\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"DoKfuME0Em0z2CG\",\"output_index\":0,\"sequence_number\":13}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"e\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"XBFbIY4Fow2p3wz\",\"output_index\":0,\"sequence_number\":14}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" test\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"JX98m2RkrWD\",\"output_index\":0,\"sequence_number\":15}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" skill\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"yWTHn4IL3k\",\"output_index\":0,\"sequence_number\":16}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" e\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"tJChmRwpGkHi7f\",\"output_index\":0,\"sequence_number\":17}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"2\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"59X6X9YSOr0tsbm\",\"output_index\":0,\"sequence_number\":18}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"e\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"YX8fGT8PwVkyRUz\",\"output_index\":0,\"sequence_number\":19}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"-fl\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"2vD42kigSA1dk\",\"output_index\":0,\"sequence_number\":20}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"ue\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"SR8lnZvAOFNlAd\",\"output_index\":0,\"sequence_number\":21}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"-s\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"e5LCrEk4SxTUxd\",\"output_index\":0,\"sequence_number\":22}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"kill\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"yQv5KXABVNyh\",\"output_index\":0,\"sequence_number\":23}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\".\\\",\\\"\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"3SJXRzWCr6dy\",\"output_index\":0,\"sequence_number\":24}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"prompt\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"jnnoMIl9VV\",\"output_index\":0,\"sequence_number\":25}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"\\\":\\\"\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"cbOKoZJQXGHZv\",\"output_index\":0,\"sequence_number\":26}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"Run\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"lOwODhO09Mw2T\",\"output_index\":0,\"sequence_number\":27}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" the\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"HJKMWHVzo3e8\",\"output_index\":0,\"sequence_number\":28}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" skill\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"pfSw7BP6rF\",\"output_index\":0,\"sequence_number\":29}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" named\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"gdjMLdp6nl\",\"output_index\":0,\"sequence_number\":30}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" \\\\\\\"\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"hb129i9TWIbbM\",\"output_index\":0,\"sequence_number\":31}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"e\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"EUDmWBk8oSvNHEU\",\"output_index\":0,\"sequence_number\":32}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"2\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"PkeI32czJqwmlmr\",\"output_index\":0,\"sequence_number\":33}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"e\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"H2UQSRf8sMZNRBG\",\"output_index\":0,\"sequence_number\":34}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"-fl\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"coZw5rkSM9llJ\",\"output_index\":0,\"sequence_number\":35}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"ue\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"RTDrspHm2QXfY9\",\"output_index\":0,\"sequence_number\":36}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"-s\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"dhLU5GXYLswOhi\",\"output_index\":0,\"sequence_number\":37}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"kill\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"PNAPCNmx4QJp\",\"output_index\":0,\"sequence_number\":38}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"\\\\\\\"\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"CpbMQHjxstrcsu\",\"output_index\":0,\"sequence_number\":39}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" with\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"KOoqv80ko3G\",\"output_index\":0,\"sequence_number\":40}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" arguments\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"jcYH3O\",\"output_index\":0,\"sequence_number\":41}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" {\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"szXf5lZHyCpilh\",\"output_index\":0,\"sequence_number\":42}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"\\\\\\\"\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"52sGZR8YVknFj3\",\"output_index\":0,\"sequence_number\":43}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"marker\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"J8ZugJCeBJ\",\"output_index\":0,\"sequence_number\":44}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"\\\\\\\":\\\\\\\"\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"2Ffk7q5BGoV\",\"output_index\":0,\"sequence_number\":45}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"SK\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"xeKflxURNDoVyb\",\"output_index\":0,\"sequence_number\":46}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"ILL\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"XjKJ0bhJcewlH\",\"output_index\":0,\"sequence_number\":47}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"_DONE\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"R4FhmYdoiCO\",\"output_index\":0,\"sequence_number\":48}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"\\\\\\\"\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"X6XdGVk1CC6NjY\",\"output_index\":0,\"sequence_number\":49}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"}.\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"YYRBPkT64KLrlE\",\"output_index\":0,\"sequence_number\":50}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" Output\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"tzFy24CSr\",\"output_index\":0,\"sequence_number\":51}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" only\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"rf2kD3jJIfw\",\"output_index\":0,\"sequence_number\":52}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" the\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"MZEAVTJ1ktEY\",\"output_index\":0,\"sequence_number\":53}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" marker\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"hsFGNt1CH\",\"output_index\":0,\"sequence_number\":54}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" and\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"7jo0OqDghkTU\",\"output_index\":0,\"sequence_number\":55}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" no\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"x7Q4KF2sTB6xe\",\"output_index\":0,\"sequence_number\":56}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" extra\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"5lWyL7VcaY\",\"output_index\":0,\"sequence_number\":57}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" text\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"P6M5oge56Cp\",\"output_index\":0,\"sequence_number\":58}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" when\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"aSZOArmUWCv\",\"output_index\":0,\"sequence_number\":59}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" finished\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"Bcwcv1j\",\"output_index\":0,\"sequence_number\":60}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\".\\\",\\\"\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"QPfZWJIaOZZK\",\"output_index\":0,\"sequence_number\":61}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"cwd\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"8CNMuVP6JkXXp\",\"output_index\":0,\"sequence_number\":62}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"\\\":\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"Zyun5ONtg0VES4\",\"output_index\":0,\"sequence_number\":63}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"\\\"/\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"0hQ1Tpvy40v6ov\",\"output_index\":0,\"sequence_number\":64}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"ver\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"AX7KPkq2wczcX\",\"output_index\":0,\"sequence_number\":65}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"cel\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"gqTRHNnYCqD98\",\"output_index\":0,\"sequence_number\":66}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/s\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"gkSS5rmctsBm7v\",\"output_index\":0,\"sequence_number\":67}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"andbox\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"Mfu7ocBSyP\",\"output_index\":0,\"sequence_number\":68}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"RYfnUiUEM7nSWVO\",\"output_index\":0,\"sequence_number\":69}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"brain\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"9TJqEA9kGKQ\",\"output_index\":0,\"sequence_number\":70}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"trust\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"lYlMUrHuRU3\",\"output_index\":0,\"sequence_number\":71}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"data\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"7rcyz6emzCww\",\"output_index\":0,\"sequence_number\":72}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"iuArNS0owWYS7ks\",\"output_index\":0,\"sequence_number\":73}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"brain\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"aO4xFHIWApy\",\"output_index\":0,\"sequence_number\":74}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"trust\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"MJOzy2JsM8I\",\"output_index\":0,\"sequence_number\":75}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"-sdk\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"sO3CIxyDjZko\",\"output_index\":0,\"sequence_number\":76}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"-j\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"NXKXqePfpHdojz\",\"output_index\":0,\"sequence_number\":77}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"avascript\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"r2Ux15z\",\"output_index\":0,\"sequence_number\":78}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/e\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"Kab8AO35yKTKmS\",\"output_index\":0,\"sequence_number\":79}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"2\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"tIH5zoiqj9CTjyo\",\"output_index\":0,\"sequence_number\":80}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"e\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"iLppUWbjIW0Alm6\",\"output_index\":0,\"sequence_number\":81}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/.\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"eBh1cQtXPmFWjX\",\"output_index\":0,\"sequence_number\":82}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"bt\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"ejJv2wp49FgCgw\",\"output_index\":0,\"sequence_number\":83}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"-t\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"Pe5APONBPL5Cz5\",\"output_index\":0,\"sequence_number\":84}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"mp\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"8L98Ly29NFwgR8\",\"output_index\":0,\"sequence_number\":85}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/run\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"zhXouMXCcpBH\",\"output_index\":0,\"sequence_number\":86}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"-\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"tfHJ3cnyfuchLJx\",\"output_index\":0,\"sequence_number\":87}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"9\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"Dj1UH61VeDE9CVJ\",\"output_index\":0,\"sequence_number\":88}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"sp\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"mEcW9oStLYq57Z\",\"output_index\":0,\"sequence_number\":89}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"u\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"da4hpEx6UsGmIdX\",\"output_index\":0,\"sequence_number\":90}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"Hg\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"wZYe5LO1qqwywa\",\"output_index\":0,\"sequence_number\":91}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/sc\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"FPrjbypLHyxRn\",\"output_index\":0,\"sequence_number\":92}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"enarios\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"pj0NPDTch\",\"output_index\":0,\"sequence_number\":93}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/fl\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"xd0ux5YOhgezS\",\"output_index\":0,\"sequence_number\":94}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"ue\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"cETdOP2IPG6g1d\",\"output_index\":0,\"sequence_number\":95}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"-in\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"MzOzX2St8G3JA\",\"output_index\":0,\"sequence_number\":96}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"strument\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"TbtAamDa\",\"output_index\":0,\"sequence_number\":97}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"ation\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"CJ8PGzQrtmy\",\"output_index\":0,\"sequence_number\":98}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"\\\"}\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"wrvfq006IzH0mA\",\"output_index\":0,\"sequence_number\":99}", - "event: response.function_call_arguments.done\ndata: {\"type\":\"response.function_call_arguments.done\",\"arguments\":\"{\\\"description\\\":\\\"Run the deterministic Flue instrumentation e2e test skill e2e-flue-skill.\\\",\\\"prompt\\\":\\\"Run the skill named \\\\\\\"e2e-flue-skill\\\\\\\" with arguments {\\\\\\\"marker\\\\\\\":\\\\\\\"SKILL_DONE\\\\\\\"}. Output only the marker and no extra text when finished.\\\",\\\"cwd\\\":\\\".\\\"}\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"output_index\":0,\"sequence_number\":100}", - "event: response.output_item.done\ndata: {\"type\":\"response.output_item.done\",\"item\":{\"id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"type\":\"function_call\",\"status\":\"completed\",\"arguments\":\"{\\\"description\\\":\\\"Run the deterministic Flue instrumentation e2e test skill e2e-flue-skill.\\\",\\\"prompt\\\":\\\"Run the skill named \\\\\\\"e2e-flue-skill\\\\\\\" with arguments {\\\\\\\"marker\\\\\\\":\\\\\\\"SKILL_DONE\\\\\\\"}. Output only the marker and no extra text when finished.\\\",\\\"cwd\\\":\\\".\\\"}\",\"call_id\":\"call_R2a5LOml0NZqq5XTcTCKfbMW\",\"name\":\"task\"},\"output_index\":0,\"sequence_number\":101}", - "event: response.completed\ndata: {\"type\":\"response.completed\",\"response\":{\"id\":\"resp_0f099b3fce0721fd016a705b63235881a393625d6974243e44\",\"object\":\"response\",\"created_at\":1785748323,\"status\":\"completed\",\"background\":false,\"completed_at\":1785748324,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":null,\"max_tool_calls\":null,\"model\":\"gpt-5.4-nano-2026-03-17\",\"moderation\":null,\"output\":[{\"id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"type\":\"function_call\",\"status\":\"completed\",\"arguments\":\"{\\\"description\\\":\\\"Run the deterministic Flue instrumentation e2e test skill e2e-flue-skill.\\\",\\\"prompt\\\":\\\"Run the skill named \\\\\\\"e2e-flue-skill\\\\\\\" with arguments {\\\\\\\"marker\\\\\\\":\\\\\\\"SKILL_DONE\\\\\\\"}. Output only the marker and no extra text when finished.\\\",\\\"cwd\\\":\\\".\\\"}\",\"call_id\":\"call_R2a5LOml0NZqq5XTcTCKfbMW\",\"name\":\"task\"}],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":\"workflow:instrumentation:01KZ3E9TWTAEK4P4P3F63HKAKC::default::sk\",\"prompt_cache_retention\":\"24h\",\"reasoning\":{\"context\":\"current_turn\",\"effort\":\"none\",\"mode\":\"standard\",\"summary\":null},\"safety_identifier\":null,\"service_tier\":\"default\",\"store\":false,\"temperature\":1,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tool_usage\":{\"image_gen\":{\"input_tokens\":0,\"input_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"output_tokens\":0,\"output_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"total_tokens\":0},\"web_search\":{\"num_requests\":0}},\"tools\":[{\"type\":\"function\",\"description\":\"Read a file or list a directory. For files, output is truncated to 2000 lines or 50KB — use offset/limit for large files. For directories, returns the list of entries.\",\"name\":\"read\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to read\"},\"offset\":{\"type\":\"number\",\"description\":\"Line number to start from (1-indexed)\"},\"limit\":{\"type\":\"number\",\"description\":\"Maximum number of lines to read\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Write content to a file. Creates the file and parent directories if they do not exist.\",\"name\":\"write\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"content\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to write\"},\"content\":{\"type\":\"string\",\"description\":\"Content to write to the file\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Edit a file using exact text replacement. The oldText must match a unique region of the file. Use replaceAll to replace all occurrences.\",\"name\":\"edit\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"oldText\",\"newText\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to edit\"},\"oldText\":{\"type\":\"string\",\"description\":\"Exact text to find (must be unique)\"},\"newText\":{\"type\":\"string\",\"description\":\"Replacement text\"},\"replaceAll\":{\"type\":\"boolean\",\"description\":\"Replace all occurrences\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Execute a bash command. Returns stdout and stderr. Output is truncated to the last 2000 lines or 50KB.\",\"name\":\"bash\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"command\"],\"properties\":{\"command\":{\"type\":\"string\",\"description\":\"Bash command to execute\"},\"timeout\":{\"type\":\"number\",\"description\":\"Timeout in seconds\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Search file contents for a regex pattern. Returns matching lines with file paths and line numbers.\",\"name\":\"grep\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Search pattern (regex)\"},\"path\":{\"type\":\"string\",\"description\":\"Directory or file to search (default: .)\"},\"include\":{\"type\":\"string\",\"description\":\"Glob filter, e.g. \\\"*.ts\\\"\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Find files by filename pattern using shell find -name semantics. Returns matching file paths.\",\"name\":\"glob\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Filename pattern, e.g. \\\"*.ts\\\"\"},\"path\":{\"type\":\"string\",\"description\":\"Directory to search in (default: .)\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Delegate a focused task to a detached child agent with its own context. Use this for independent research, file exploration, or parallel work. The task returns only its final answer to this conversation. No subagents are currently defined.\",\"name\":\"task\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"prompt\"],\"properties\":{\"description\":{\"type\":\"string\",\"description\":\"Short human-readable label for the delegated work\"},\"prompt\":{\"type\":\"string\",\"description\":\"Focused instructions for the child agent\"},\"agent\":{\"type\":\"string\",\"description\":\"Declared subagent to use for the child agent\"},\"cwd\":{\"type\":\"string\",\"description\":\"Working directory for the child agent. AGENTS.md and skills are discovered from here.\"}}},\"strict\":false}],\"top_logprobs\":0,\"top_p\":0.98,\"truncation\":\"disabled\",\"usage\":{\"input_tokens\":847,\"input_tokens_details\":{\"cache_write_tokens\":0,\"cached_tokens\":0},\"output_tokens\":109,\"output_tokens_details\":{\"reasoning_tokens\":0},\"total_tokens\":956},\"user\":null,\"metadata\":{}},\"sequence_number\":102}" - ], - "kind": "sse" - }, - "headers": { - "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", - "alt-svc": "h3=\":443\"; ma=86400", - "cf-cache-status": "DYNAMIC", - "cf-ray": "a25432cb5fc97bb6-IAD", - "connection": "keep-alive", - "content-type": "text/event-stream; charset=utf-8", - "date": "Mon, 03 Aug 2026 09:12:03 GMT", - "openai-organization": "braintrust-data", - "openai-processing-ms": "206", - "openai-project": "proj_vsCSXafhhByzWOThMrJcZiw9", - "openai-version": "2020-10-01", - "server": "cloudflare", - "set-cookie": "[REDACTED]", - "strict-transport-security": "max-age=31536000; includeSubDomains; preload", - "transfer-encoding": "chunked", - "x-content-type-options": "nosniff", - "x-ratelimit-limit-requests": "30000", - "x-ratelimit-limit-tokens": "180000000", - "x-ratelimit-remaining-requests": "29999", - "x-ratelimit-remaining-tokens": "179998698", - "x-ratelimit-reset-requests": "2ms", - "x-ratelimit-reset-tokens": "0s", - "x-request-id": "req_4f8a543160de4adea11337a977a14713" - }, - "status": 200, - "statusText": "OK" - } - }, - { - "callIndex": 5, - "id": "341675a0a1a88c63", - "matchKey": "POST api.openai.com/v1/responses", - "recordedAt": "2026-08-03T09:12:05.801Z", - "request": { - "body": { - "kind": "json", - "value": { - "input": [ - { - "content": "You are running in headless mode with no human operator. Work autonomously — never ask questions, never wait for user input. Make your best judgment and proceed independently.\n\nYou are a deterministic Flue instrumentation test agent. Follow user instructions exactly. When asked for a marker, output only that marker and no extra text. When running a local skill file, read it yourself and do not delegate it to a task.\n\n## Available Skills\n\nThe following skills provide specialized instructions for specific tasks. When a task matches a skill description, activate that skill before proceeding so its full instructions are loaded. Skill instructions and supporting resources stay lazy until activation or explicit file reads.\n\n- **e2e-flue-skill** — Deterministic skill used by the Braintrust Flue instrumentation e2e test.\n\nDate: Mon, Aug 3, 2026\nWorking directory: .\n\nDirectory structure:\n.agents\n.flue\n.flue-build\n__cassettes__\n__snapshots__\nassertions.ts\ncassette-filter.mjs\nconstants.mjs\ndist\nnode_modules\npackage.json\npnpm-lock.yaml\nscenario.cli.mjs\nscenario.impl.mjs\nscenario.mjs\nscenario.test.ts\nscenario.ts\nscenario.v2.ts\nv2-assertions.ts\nversions", - "role": "developer" - }, - { - "content": [ - { - "text": "Run the skill named \"e2e-flue-skill\" with arguments {\"marker\":\"SKILL_DONE\"}. Output only the marker and no extra text when finished.", - "type": "input_text" - } - ], - "role": "user" - } - ], - "model": "gpt-5.4-nano", - "prompt_cache_key": "[REDACTED]", - "reasoning": { - "effort": "none" - }, - "store": false, - "stream": true, - "tools": [ - { - "description": "Read a file or list a directory. For files, output is truncated to 2000 lines or 50KB — use offset/limit for large files. For directories, returns the list of entries.", - "name": "read", - "parameters": { - "properties": { - "limit": { - "description": "Maximum number of lines to read", - "type": "number" - }, - "offset": { - "description": "Line number to start from (1-indexed)", - "type": "number" - }, - "path": { - "description": "Path to the file to read", - "type": "string" - } - }, - "required": ["path"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Write content to a file. Creates the file and parent directories if they do not exist.", - "name": "write", - "parameters": { - "properties": { - "content": { - "description": "Content to write to the file", - "type": "string" - }, - "path": { - "description": "Path to the file to write", - "type": "string" - } - }, - "required": ["path", "content"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Edit a file using exact text replacement. The oldText must match a unique region of the file. Use replaceAll to replace all occurrences.", - "name": "edit", - "parameters": { - "properties": { - "newText": { - "description": "Replacement text", - "type": "string" - }, - "oldText": { - "description": "Exact text to find (must be unique)", - "type": "string" - }, - "path": { - "description": "Path to the file to edit", - "type": "string" - }, - "replaceAll": { - "description": "Replace all occurrences", - "type": "boolean" - } - }, - "required": ["path", "oldText", "newText"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Execute a bash command. Returns stdout and stderr. Output is truncated to the last 2000 lines or 50KB.", - "name": "bash", - "parameters": { - "properties": { - "command": { - "description": "Bash command to execute", - "type": "string" - }, - "timeout": { - "description": "Timeout in seconds", - "type": "number" - } - }, - "required": ["command"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Search file contents for a regex pattern. Returns matching lines with file paths and line numbers.", - "name": "grep", - "parameters": { - "properties": { - "include": { - "description": "Glob filter, e.g. \"*.ts\"", - "type": "string" - }, - "path": { - "description": "Directory or file to search (default: .)", - "type": "string" - }, - "pattern": { - "description": "Search pattern (regex)", - "type": "string" - } - }, - "required": ["pattern"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Find files by filename pattern using shell find -name semantics. Returns matching file paths.", - "name": "glob", - "parameters": { - "properties": { - "path": { - "description": "Directory to search in (default: .)", - "type": "string" - }, - "pattern": { - "description": "Filename pattern, e.g. \"*.ts\"", - "type": "string" - } - }, - "required": ["pattern"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Delegate a focused task to a detached child agent with its own context. Use this for independent research, file exploration, or parallel work. The task returns only its final answer to this conversation. No subagents are currently defined.", - "name": "task", - "parameters": { - "properties": { - "agent": { - "description": "Declared subagent to use for the child agent", - "type": "string" - }, - "cwd": { - "description": "Working directory for the child agent. AGENTS.md and skills are discovered from here.", - "type": "string" - }, - "description": { - "description": "Short human-readable label for the delegated work", - "type": "string" - }, - "prompt": { - "description": "Focused instructions for the child agent", - "type": "string" - } - }, - "required": ["prompt"], - "type": "object" - }, - "strict": false, - "type": "function" - } - ] - } - }, - "headers": {}, - "method": "POST", - "url": "https://api.openai.com/v1/responses" - }, - "response": { - "body": { - "chunks": [ - "event: response.created\ndata: {\"type\":\"response.created\",\"response\":{\"id\":\"resp_054a16a2f2ea3217016a705b648f7081a3b96733816812acb8\",\"object\":\"response\",\"created_at\":1785748324,\"status\":\"in_progress\",\"background\":false,\"completed_at\":null,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":null,\"max_tool_calls\":null,\"model\":\"gpt-5.4-nano-2026-03-17\",\"moderation\":null,\"output\":[],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":\"workflow:instrumentation:01KZ3E9TWTAEK4P4P3F63HKAKC::default::ta\",\"prompt_cache_retention\":\"24h\",\"reasoning\":{\"context\":\"current_turn\",\"effort\":\"none\",\"mode\":\"standard\",\"summary\":null},\"safety_identifier\":null,\"service_tier\":\"auto\",\"store\":false,\"temperature\":1,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tool_usage\":{\"image_gen\":{\"input_tokens\":0,\"input_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"output_tokens\":0,\"output_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"total_tokens\":0},\"web_search\":{\"num_requests\":0}},\"tools\":[{\"type\":\"function\",\"description\":\"Read a file or list a directory. For files, output is truncated to 2000 lines or 50KB — use offset/limit for large files. For directories, returns the list of entries.\",\"name\":\"read\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to read\"},\"offset\":{\"type\":\"number\",\"description\":\"Line number to start from (1-indexed)\"},\"limit\":{\"type\":\"number\",\"description\":\"Maximum number of lines to read\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Write content to a file. Creates the file and parent directories if they do not exist.\",\"name\":\"write\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"content\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to write\"},\"content\":{\"type\":\"string\",\"description\":\"Content to write to the file\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Edit a file using exact text replacement. The oldText must match a unique region of the file. Use replaceAll to replace all occurrences.\",\"name\":\"edit\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"oldText\",\"newText\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to edit\"},\"oldText\":{\"type\":\"string\",\"description\":\"Exact text to find (must be unique)\"},\"newText\":{\"type\":\"string\",\"description\":\"Replacement text\"},\"replaceAll\":{\"type\":\"boolean\",\"description\":\"Replace all occurrences\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Execute a bash command. Returns stdout and stderr. Output is truncated to the last 2000 lines or 50KB.\",\"name\":\"bash\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"command\"],\"properties\":{\"command\":{\"type\":\"string\",\"description\":\"Bash command to execute\"},\"timeout\":{\"type\":\"number\",\"description\":\"Timeout in seconds\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Search file contents for a regex pattern. Returns matching lines with file paths and line numbers.\",\"name\":\"grep\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Search pattern (regex)\"},\"path\":{\"type\":\"string\",\"description\":\"Directory or file to search (default: .)\"},\"include\":{\"type\":\"string\",\"description\":\"Glob filter, e.g. \\\"*.ts\\\"\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Find files by filename pattern using shell find -name semantics. Returns matching file paths.\",\"name\":\"glob\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Filename pattern, e.g. \\\"*.ts\\\"\"},\"path\":{\"type\":\"string\",\"description\":\"Directory to search in (default: .)\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Delegate a focused task to a detached child agent with its own context. Use this for independent research, file exploration, or parallel work. The task returns only its final answer to this conversation. No subagents are currently defined.\",\"name\":\"task\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"prompt\"],\"properties\":{\"description\":{\"type\":\"string\",\"description\":\"Short human-readable label for the delegated work\"},\"prompt\":{\"type\":\"string\",\"description\":\"Focused instructions for the child agent\"},\"agent\":{\"type\":\"string\",\"description\":\"Declared subagent to use for the child agent\"},\"cwd\":{\"type\":\"string\",\"description\":\"Working directory for the child agent. AGENTS.md and skills are discovered from here.\"}}},\"strict\":false}],\"top_logprobs\":0,\"top_p\":0.98,\"truncation\":\"disabled\",\"usage\":null,\"user\":null,\"metadata\":{}},\"sequence_number\":0}", - "event: response.in_progress\ndata: {\"type\":\"response.in_progress\",\"response\":{\"id\":\"resp_054a16a2f2ea3217016a705b648f7081a3b96733816812acb8\",\"object\":\"response\",\"created_at\":1785748324,\"status\":\"in_progress\",\"background\":false,\"completed_at\":null,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":null,\"max_tool_calls\":null,\"model\":\"gpt-5.4-nano-2026-03-17\",\"moderation\":null,\"output\":[],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":\"workflow:instrumentation:01KZ3E9TWTAEK4P4P3F63HKAKC::default::ta\",\"prompt_cache_retention\":\"24h\",\"reasoning\":{\"context\":\"current_turn\",\"effort\":\"none\",\"mode\":\"standard\",\"summary\":null},\"safety_identifier\":null,\"service_tier\":\"auto\",\"store\":false,\"temperature\":1,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tool_usage\":{\"image_gen\":{\"input_tokens\":0,\"input_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"output_tokens\":0,\"output_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"total_tokens\":0},\"web_search\":{\"num_requests\":0}},\"tools\":[{\"type\":\"function\",\"description\":\"Read a file or list a directory. For files, output is truncated to 2000 lines or 50KB — use offset/limit for large files. For directories, returns the list of entries.\",\"name\":\"read\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to read\"},\"offset\":{\"type\":\"number\",\"description\":\"Line number to start from (1-indexed)\"},\"limit\":{\"type\":\"number\",\"description\":\"Maximum number of lines to read\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Write content to a file. Creates the file and parent directories if they do not exist.\",\"name\":\"write\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"content\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to write\"},\"content\":{\"type\":\"string\",\"description\":\"Content to write to the file\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Edit a file using exact text replacement. The oldText must match a unique region of the file. Use replaceAll to replace all occurrences.\",\"name\":\"edit\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"oldText\",\"newText\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to edit\"},\"oldText\":{\"type\":\"string\",\"description\":\"Exact text to find (must be unique)\"},\"newText\":{\"type\":\"string\",\"description\":\"Replacement text\"},\"replaceAll\":{\"type\":\"boolean\",\"description\":\"Replace all occurrences\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Execute a bash command. Returns stdout and stderr. Output is truncated to the last 2000 lines or 50KB.\",\"name\":\"bash\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"command\"],\"properties\":{\"command\":{\"type\":\"string\",\"description\":\"Bash command to execute\"},\"timeout\":{\"type\":\"number\",\"description\":\"Timeout in seconds\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Search file contents for a regex pattern. Returns matching lines with file paths and line numbers.\",\"name\":\"grep\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Search pattern (regex)\"},\"path\":{\"type\":\"string\",\"description\":\"Directory or file to search (default: .)\"},\"include\":{\"type\":\"string\",\"description\":\"Glob filter, e.g. \\\"*.ts\\\"\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Find files by filename pattern using shell find -name semantics. Returns matching file paths.\",\"name\":\"glob\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Filename pattern, e.g. \\\"*.ts\\\"\"},\"path\":{\"type\":\"string\",\"description\":\"Directory to search in (default: .)\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Delegate a focused task to a detached child agent with its own context. Use this for independent research, file exploration, or parallel work. The task returns only its final answer to this conversation. No subagents are currently defined.\",\"name\":\"task\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"prompt\"],\"properties\":{\"description\":{\"type\":\"string\",\"description\":\"Short human-readable label for the delegated work\"},\"prompt\":{\"type\":\"string\",\"description\":\"Focused instructions for the child agent\"},\"agent\":{\"type\":\"string\",\"description\":\"Declared subagent to use for the child agent\"},\"cwd\":{\"type\":\"string\",\"description\":\"Working directory for the child agent. AGENTS.md and skills are discovered from here.\"}}},\"strict\":false}],\"top_logprobs\":0,\"top_p\":0.98,\"truncation\":\"disabled\",\"usage\":null,\"user\":null,\"metadata\":{}},\"sequence_number\":1}", - "event: response.output_item.added\ndata: {\"type\":\"response.output_item.added\",\"item\":{\"id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"type\":\"function_call\",\"status\":\"in_progress\",\"arguments\":\"\",\"call_id\":\"call_w6pyYZgnhlSOAXul75DAG1Cj\",\"name\":\"task\"},\"output_index\":0,\"sequence_number\":2}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"{\\\"\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"ljpLbllVFarTqw\",\"output_index\":0,\"sequence_number\":3}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"description\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"dI5yK\",\"output_index\":0,\"sequence_number\":4}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"\\\":\\\"\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"gaE80vRkyue6q\",\"output_index\":0,\"sequence_number\":5}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"Run\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"1yh2B3GwgUIkU\",\"output_index\":0,\"sequence_number\":6}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" e\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"Aoc2XDyvRflwpK\",\"output_index\":0,\"sequence_number\":7}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"2\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"gKDO3Feq1OjvKMT\",\"output_index\":0,\"sequence_number\":8}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"e\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"d2O2IlHGALtgxiz\",\"output_index\":0,\"sequence_number\":9}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"-fl\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"c9m5GXjLlVm1X\",\"output_index\":0,\"sequence_number\":10}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"ue\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"NUY5MBuNzz8sZQ\",\"output_index\":0,\"sequence_number\":11}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"-s\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"9Pwv64hhPdkCsg\",\"output_index\":0,\"sequence_number\":12}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"kill\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"iOoKz15nYfXp\",\"output_index\":0,\"sequence_number\":13}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" with\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"UCahcEprOGt\",\"output_index\":0,\"sequence_number\":14}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" marker\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"TaOkeLqZF\",\"output_index\":0,\"sequence_number\":15}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" SK\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"VxlrD2puv3ABI\",\"output_index\":0,\"sequence_number\":16}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"ILL\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"DebTGrlabFRQY\",\"output_index\":0,\"sequence_number\":17}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"_DONE\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"Y6K3aF28EHz\",\"output_index\":0,\"sequence_number\":18}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"\\\",\\\"\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"py2Pd0lnhbcOC\",\"output_index\":0,\"sequence_number\":19}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"prompt\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"PM2ySL1u5c\",\"output_index\":0,\"sequence_number\":20}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"\\\":\\\"\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"GEFeUIpfX9F6O\",\"output_index\":0,\"sequence_number\":21}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"Run\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"g8vFm9KNQk3gN\",\"output_index\":0,\"sequence_number\":22}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" the\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"cXxSzWyVKRHs\",\"output_index\":0,\"sequence_number\":23}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" skill\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"bwbVemN5vW\",\"output_index\":0,\"sequence_number\":24}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" named\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"4DdymViVIi\",\"output_index\":0,\"sequence_number\":25}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" \\\\\\\"\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"9QwzJKU8yuXI1\",\"output_index\":0,\"sequence_number\":26}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"e\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"qc8c9FjKJwS85NJ\",\"output_index\":0,\"sequence_number\":27}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"2\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"ewNZxFO4TdAgDUM\",\"output_index\":0,\"sequence_number\":28}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"e\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"Y8xyYUu8EtAbcT1\",\"output_index\":0,\"sequence_number\":29}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"-fl\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"88SoGMEyRHz8M\",\"output_index\":0,\"sequence_number\":30}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"ue\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"DpP2H8WkpDUzag\",\"output_index\":0,\"sequence_number\":31}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"-s\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"5jctq9RddqcIt8\",\"output_index\":0,\"sequence_number\":32}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"kill\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"xpmcaPHyYGYa\",\"output_index\":0,\"sequence_number\":33}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"\\\\\\\"\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"kumygUHU0Bd682\",\"output_index\":0,\"sequence_number\":34}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" with\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"1aD8huNzRAg\",\"output_index\":0,\"sequence_number\":35}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" arguments\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"u62pY1\",\"output_index\":0,\"sequence_number\":36}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" {\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"DlufXJUKJUw7nw\",\"output_index\":0,\"sequence_number\":37}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"\\\\\\\"\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"dqIkwjYJ3Ruj51\",\"output_index\":0,\"sequence_number\":38}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"marker\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"stUcL5JVqc\",\"output_index\":0,\"sequence_number\":39}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"\\\\\\\":\\\\\\\"\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"cUfPJq2cus1\",\"output_index\":0,\"sequence_number\":40}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"SK\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"ZzPWTx2aFPwBYM\",\"output_index\":0,\"sequence_number\":41}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"ILL\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"Hbw389fTXAJOT\",\"output_index\":0,\"sequence_number\":42}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"_DONE\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"ALgrsM7YeM5\",\"output_index\":0,\"sequence_number\":43}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"\\\\\\\"\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"D4fcykCofi3ap6\",\"output_index\":0,\"sequence_number\":44}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"}.\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"79o07UPm9B29cw\",\"output_index\":0,\"sequence_number\":45}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" Ensure\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"1JXF9XGzd\",\"output_index\":0,\"sequence_number\":46}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" completion\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"vfueK\",\"output_index\":0,\"sequence_number\":47}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" and\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"VlKAzFGFtrcw\",\"output_index\":0,\"sequence_number\":48}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" report\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"wBeH1rhXr\",\"output_index\":0,\"sequence_number\":49}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" the\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"6EO4vypCMr7q\",\"output_index\":0,\"sequence_number\":50}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" final\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"0pF35NFSxj\",\"output_index\":0,\"sequence_number\":51}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" output\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"Fyh51oBFK\",\"output_index\":0,\"sequence_number\":52}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" marker\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"2fdE8CEMT\",\"output_index\":0,\"sequence_number\":53}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" only\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"Oi1kYTafLLr\",\"output_index\":0,\"sequence_number\":54}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\".\\\",\\\"\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"JHNARwrOOg28\",\"output_index\":0,\"sequence_number\":55}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"agent\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"JujpXurFNFJ\",\"output_index\":0,\"sequence_number\":56}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"\\\":\\\"\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"7UtnbpjQ9Vd5r\",\"output_index\":0,\"sequence_number\":57}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"e\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"lUMgv4pWgFguMgN\",\"output_index\":0,\"sequence_number\":58}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"2\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"lquqyIOhUG8FFqb\",\"output_index\":0,\"sequence_number\":59}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"e\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"z1aAXV10WTtEwfO\",\"output_index\":0,\"sequence_number\":60}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"-fl\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"z29NnUuO8awvU\",\"output_index\":0,\"sequence_number\":61}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"ue\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"vjUb5xZd4tz6Nc\",\"output_index\":0,\"sequence_number\":62}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"-s\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"S79uypIplW7mcH\",\"output_index\":0,\"sequence_number\":63}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"kill\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"GgbDCJps9iY5\",\"output_index\":0,\"sequence_number\":64}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"\\\",\\\"\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"vCCHCimmRyAgx\",\"output_index\":0,\"sequence_number\":65}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"cwd\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"AnLDQiNay8ENp\",\"output_index\":0,\"sequence_number\":66}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"\\\":\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"6YVXzJiSMPKoqe\",\"output_index\":0,\"sequence_number\":67}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"\\\"/\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"2iVYgZDItlXYju\",\"output_index\":0,\"sequence_number\":68}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"ver\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"p8iffYf14P4rP\",\"output_index\":0,\"sequence_number\":69}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"cel\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"jHIrIgM5myBm6\",\"output_index\":0,\"sequence_number\":70}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/s\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"Yud9IuaLw1CQTi\",\"output_index\":0,\"sequence_number\":71}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"andbox\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"sxVNzIOgmI\",\"output_index\":0,\"sequence_number\":72}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"BFftBcl4q4mRiic\",\"output_index\":0,\"sequence_number\":73}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"brain\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"bvKWRZsGF3z\",\"output_index\":0,\"sequence_number\":74}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"trust\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"8fCBYfWjld6\",\"output_index\":0,\"sequence_number\":75}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"data\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"bWCV0wB1yOIj\",\"output_index\":0,\"sequence_number\":76}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"gMLQ5nGNZU5RQcA\",\"output_index\":0,\"sequence_number\":77}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"brain\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"HbDV4gJR67J\",\"output_index\":0,\"sequence_number\":78}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"trust\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"fY6Fye7lRfU\",\"output_index\":0,\"sequence_number\":79}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"-sdk\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"nDcCSPqarvNr\",\"output_index\":0,\"sequence_number\":80}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"-j\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"CDVkHgmi29Aam7\",\"output_index\":0,\"sequence_number\":81}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"avascript\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"yMH6cPx\",\"output_index\":0,\"sequence_number\":82}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/e\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"zoEfWKIxJ3vi4r\",\"output_index\":0,\"sequence_number\":83}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"2\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"ti33GAqnm3z55of\",\"output_index\":0,\"sequence_number\":84}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"e\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"iKabCz0geb00km7\",\"output_index\":0,\"sequence_number\":85}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/.\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"kQaYnx9LpaZ4YA\",\"output_index\":0,\"sequence_number\":86}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"bt\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"rHzMr4AWuPUfim\",\"output_index\":0,\"sequence_number\":87}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"-t\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"nJma8KAPeFozgD\",\"output_index\":0,\"sequence_number\":88}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"mp\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"bJV57iCDmqv2VA\",\"output_index\":0,\"sequence_number\":89}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/run\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"IpVQRAjXMPE4\",\"output_index\":0,\"sequence_number\":90}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"-\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"pzJyXElP7ou0ovn\",\"output_index\":0,\"sequence_number\":91}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"9\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"1ZlIYGuEL0CGzns\",\"output_index\":0,\"sequence_number\":92}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"sp\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"QVS1D8jJbqlZ7X\",\"output_index\":0,\"sequence_number\":93}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"u\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"2mhaJD9do70hkwJ\",\"output_index\":0,\"sequence_number\":94}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"Hg\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"dXooQVlSHUeRZP\",\"output_index\":0,\"sequence_number\":95}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/sc\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"JYgibM5tWMAjH\",\"output_index\":0,\"sequence_number\":96}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"enarios\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"JH3lHc5hu\",\"output_index\":0,\"sequence_number\":97}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/fl\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"S7XC3UsMmTfuf\",\"output_index\":0,\"sequence_number\":98}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"ue\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"xWgqKM1XAI7Gka\",\"output_index\":0,\"sequence_number\":99}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"-in\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"7l79NEJnKe3pM\",\"output_index\":0,\"sequence_number\":100}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"strument\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"F9vfZGnW\",\"output_index\":0,\"sequence_number\":101}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"ation\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"iNKeVMoTNBN\",\"output_index\":0,\"sequence_number\":102}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"\\\"}\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"dJaOqvNAHb8EH5\",\"output_index\":0,\"sequence_number\":103}", - "event: response.function_call_arguments.done\ndata: {\"type\":\"response.function_call_arguments.done\",\"arguments\":\"{\\\"description\\\":\\\"Run e2e-flue-skill with marker SKILL_DONE\\\",\\\"prompt\\\":\\\"Run the skill named \\\\\\\"e2e-flue-skill\\\\\\\" with arguments {\\\\\\\"marker\\\\\\\":\\\\\\\"SKILL_DONE\\\\\\\"}. Ensure completion and report the final output marker only.\\\",\\\"agent\\\":\\\"e2e-flue-skill\\\",\\\"cwd\\\":\\\".\\\"}\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"output_index\":0,\"sequence_number\":104}", - "event: response.output_item.done\ndata: {\"type\":\"response.output_item.done\",\"item\":{\"id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"type\":\"function_call\",\"status\":\"completed\",\"arguments\":\"{\\\"description\\\":\\\"Run e2e-flue-skill with marker SKILL_DONE\\\",\\\"prompt\\\":\\\"Run the skill named \\\\\\\"e2e-flue-skill\\\\\\\" with arguments {\\\\\\\"marker\\\\\\\":\\\\\\\"SKILL_DONE\\\\\\\"}. Ensure completion and report the final output marker only.\\\",\\\"agent\\\":\\\"e2e-flue-skill\\\",\\\"cwd\\\":\\\".\\\"}\",\"call_id\":\"call_w6pyYZgnhlSOAXul75DAG1Cj\",\"name\":\"task\"},\"output_index\":0,\"sequence_number\":105}", - "event: response.completed\ndata: {\"type\":\"response.completed\",\"response\":{\"id\":\"resp_054a16a2f2ea3217016a705b648f7081a3b96733816812acb8\",\"object\":\"response\",\"created_at\":1785748324,\"status\":\"completed\",\"background\":false,\"completed_at\":1785748325,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":null,\"max_tool_calls\":null,\"model\":\"gpt-5.4-nano-2026-03-17\",\"moderation\":null,\"output\":[{\"id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"type\":\"function_call\",\"status\":\"completed\",\"arguments\":\"{\\\"description\\\":\\\"Run e2e-flue-skill with marker SKILL_DONE\\\",\\\"prompt\\\":\\\"Run the skill named \\\\\\\"e2e-flue-skill\\\\\\\" with arguments {\\\\\\\"marker\\\\\\\":\\\\\\\"SKILL_DONE\\\\\\\"}. Ensure completion and report the final output marker only.\\\",\\\"agent\\\":\\\"e2e-flue-skill\\\",\\\"cwd\\\":\\\".\\\"}\",\"call_id\":\"call_w6pyYZgnhlSOAXul75DAG1Cj\",\"name\":\"task\"}],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":\"workflow:instrumentation:01KZ3E9TWTAEK4P4P3F63HKAKC::default::ta\",\"prompt_cache_retention\":\"24h\",\"reasoning\":{\"context\":\"current_turn\",\"effort\":\"none\",\"mode\":\"standard\",\"summary\":null},\"safety_identifier\":null,\"service_tier\":\"default\",\"store\":false,\"temperature\":1,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tool_usage\":{\"image_gen\":{\"input_tokens\":0,\"input_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"output_tokens\":0,\"output_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"total_tokens\":0},\"web_search\":{\"num_requests\":0}},\"tools\":[{\"type\":\"function\",\"description\":\"Read a file or list a directory. For files, output is truncated to 2000 lines or 50KB — use offset/limit for large files. For directories, returns the list of entries.\",\"name\":\"read\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to read\"},\"offset\":{\"type\":\"number\",\"description\":\"Line number to start from (1-indexed)\"},\"limit\":{\"type\":\"number\",\"description\":\"Maximum number of lines to read\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Write content to a file. Creates the file and parent directories if they do not exist.\",\"name\":\"write\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"content\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to write\"},\"content\":{\"type\":\"string\",\"description\":\"Content to write to the file\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Edit a file using exact text replacement. The oldText must match a unique region of the file. Use replaceAll to replace all occurrences.\",\"name\":\"edit\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"oldText\",\"newText\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to edit\"},\"oldText\":{\"type\":\"string\",\"description\":\"Exact text to find (must be unique)\"},\"newText\":{\"type\":\"string\",\"description\":\"Replacement text\"},\"replaceAll\":{\"type\":\"boolean\",\"description\":\"Replace all occurrences\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Execute a bash command. Returns stdout and stderr. Output is truncated to the last 2000 lines or 50KB.\",\"name\":\"bash\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"command\"],\"properties\":{\"command\":{\"type\":\"string\",\"description\":\"Bash command to execute\"},\"timeout\":{\"type\":\"number\",\"description\":\"Timeout in seconds\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Search file contents for a regex pattern. Returns matching lines with file paths and line numbers.\",\"name\":\"grep\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Search pattern (regex)\"},\"path\":{\"type\":\"string\",\"description\":\"Directory or file to search (default: .)\"},\"include\":{\"type\":\"string\",\"description\":\"Glob filter, e.g. \\\"*.ts\\\"\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Find files by filename pattern using shell find -name semantics. Returns matching file paths.\",\"name\":\"glob\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Filename pattern, e.g. \\\"*.ts\\\"\"},\"path\":{\"type\":\"string\",\"description\":\"Directory to search in (default: .)\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Delegate a focused task to a detached child agent with its own context. Use this for independent research, file exploration, or parallel work. The task returns only its final answer to this conversation. No subagents are currently defined.\",\"name\":\"task\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"prompt\"],\"properties\":{\"description\":{\"type\":\"string\",\"description\":\"Short human-readable label for the delegated work\"},\"prompt\":{\"type\":\"string\",\"description\":\"Focused instructions for the child agent\"},\"agent\":{\"type\":\"string\",\"description\":\"Declared subagent to use for the child agent\"},\"cwd\":{\"type\":\"string\",\"description\":\"Working directory for the child agent. AGENTS.md and skills are discovered from here.\"}}},\"strict\":false}],\"top_logprobs\":0,\"top_p\":0.98,\"truncation\":\"disabled\",\"usage\":{\"input_tokens\":855,\"input_tokens_details\":{\"cache_write_tokens\":0,\"cached_tokens\":0},\"output_tokens\":113,\"output_tokens_details\":{\"reasoning_tokens\":0},\"total_tokens\":968},\"user\":null,\"metadata\":{}},\"sequence_number\":106}" - ], - "kind": "sse" - }, - "headers": { - "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", - "alt-svc": "h3=\":443\"; ma=86400", - "cf-cache-status": "DYNAMIC", - "cf-ray": "a25432d37fd37bb6-IAD", - "connection": "keep-alive", - "content-type": "text/event-stream; charset=utf-8", - "date": "Mon, 03 Aug 2026 09:12:04 GMT", - "openai-organization": "braintrust-data", - "openai-processing-ms": "268", - "openai-project": "proj_vsCSXafhhByzWOThMrJcZiw9", - "openai-version": "2020-10-01", - "server": "cloudflare", - "set-cookie": "[REDACTED]", - "strict-transport-security": "max-age=31536000; includeSubDomains; preload", - "transfer-encoding": "chunked", - "x-content-type-options": "nosniff", - "x-ratelimit-limit-requests": "30000", - "x-ratelimit-limit-tokens": "180000000", - "x-ratelimit-remaining-requests": "29999", - "x-ratelimit-remaining-tokens": "179998692", - "x-ratelimit-reset-requests": "2ms", - "x-ratelimit-reset-tokens": "0s", - "x-request-id": "req_bf6f5e02b3da40c19ef5905452f8e5f6" - }, - "status": 200, - "statusText": "OK" - } - }, - { - "callIndex": 6, - "id": "cbb9070d68f84d1b", - "matchKey": "POST api.openai.com/v1/responses", - "recordedAt": "2026-08-03T09:12:06.856Z", - "request": { - "body": { - "kind": "json", - "value": { - "input": [ - { - "content": "You are running in headless mode with no human operator. Work autonomously — never ask questions, never wait for user input. Make your best judgment and proceed independently.\n\nYou are a deterministic Flue instrumentation test agent. Follow user instructions exactly. When asked for a marker, output only that marker and no extra text. When running a local skill file, read it yourself and do not delegate it to a task.\n\n## Available Skills\n\nThe following skills provide specialized instructions for specific tasks. When a task matches a skill description, activate that skill before proceeding so its full instructions are loaded. Skill instructions and supporting resources stay lazy until activation or explicit file reads.\n\n- **e2e-flue-skill** — Deterministic skill used by the Braintrust Flue instrumentation e2e test.\n\nDate: Mon, Aug 3, 2026\nWorking directory: .\n\nDirectory structure:\n.agents\n.flue\n.flue-build\n__cassettes__\n__snapshots__\nassertions.ts\ncassette-filter.mjs\nconstants.mjs\ndist\nnode_modules\npackage.json\npnpm-lock.yaml\nscenario.cli.mjs\nscenario.impl.mjs\nscenario.mjs\nscenario.test.ts\nscenario.ts\nscenario.v2.ts\nv2-assertions.ts\nversions", - "role": "developer" - }, - { - "content": [ - { - "text": "Run the skill named \"e2e-flue-skill\" with arguments {\"marker\":\"SKILL_DONE\"}. Output only the marker and no extra text when finished.", - "type": "input_text" - } - ], - "role": "user" - }, - { - "arguments": "{\"description\":\"Run e2e-flue-skill with marker SKILL_DONE\",\"prompt\":\"Run the skill named \\\"e2e-flue-skill\\\" with arguments {\\\"marker\\\":\\\"SKILL_DONE\\\"}. Ensure completion and report the final output marker only.\",\"agent\":\"e2e-flue-skill\",\"cwd\":\".\"}", - "call_id": "call_w6pyYZgnhlSOAXul75DAG1Cj", - "id": "fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6", - "name": "task", - "type": "function_call" - }, - { - "call_id": "call_w6pyYZgnhlSOAXul75DAG1Cj", - "output": "[flue] Subagent \"e2e-flue-skill\" is not declared. Available: (none).", - "type": "function_call_output" - } - ], - "model": "gpt-5.4-nano", - "prompt_cache_key": "[REDACTED]", - "reasoning": { - "effort": "none" - }, - "store": false, - "stream": true, - "tools": [ - { - "description": "Read a file or list a directory. For files, output is truncated to 2000 lines or 50KB — use offset/limit for large files. For directories, returns the list of entries.", - "name": "read", - "parameters": { - "properties": { - "limit": { - "description": "Maximum number of lines to read", - "type": "number" - }, - "offset": { - "description": "Line number to start from (1-indexed)", - "type": "number" - }, - "path": { - "description": "Path to the file to read", - "type": "string" - } - }, - "required": ["path"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Write content to a file. Creates the file and parent directories if they do not exist.", - "name": "write", - "parameters": { - "properties": { - "content": { - "description": "Content to write to the file", - "type": "string" - }, - "path": { - "description": "Path to the file to write", - "type": "string" - } - }, - "required": ["path", "content"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Edit a file using exact text replacement. The oldText must match a unique region of the file. Use replaceAll to replace all occurrences.", - "name": "edit", - "parameters": { - "properties": { - "newText": { - "description": "Replacement text", - "type": "string" - }, - "oldText": { - "description": "Exact text to find (must be unique)", - "type": "string" - }, - "path": { - "description": "Path to the file to edit", - "type": "string" - }, - "replaceAll": { - "description": "Replace all occurrences", - "type": "boolean" - } - }, - "required": ["path", "oldText", "newText"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Execute a bash command. Returns stdout and stderr. Output is truncated to the last 2000 lines or 50KB.", - "name": "bash", - "parameters": { - "properties": { - "command": { - "description": "Bash command to execute", - "type": "string" - }, - "timeout": { - "description": "Timeout in seconds", - "type": "number" - } - }, - "required": ["command"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Search file contents for a regex pattern. Returns matching lines with file paths and line numbers.", - "name": "grep", - "parameters": { - "properties": { - "include": { - "description": "Glob filter, e.g. \"*.ts\"", - "type": "string" - }, - "path": { - "description": "Directory or file to search (default: .)", - "type": "string" - }, - "pattern": { - "description": "Search pattern (regex)", - "type": "string" - } - }, - "required": ["pattern"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Find files by filename pattern using shell find -name semantics. Returns matching file paths.", - "name": "glob", - "parameters": { - "properties": { - "path": { - "description": "Directory to search in (default: .)", - "type": "string" - }, - "pattern": { - "description": "Filename pattern, e.g. \"*.ts\"", - "type": "string" - } - }, - "required": ["pattern"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Delegate a focused task to a detached child agent with its own context. Use this for independent research, file exploration, or parallel work. The task returns only its final answer to this conversation. No subagents are currently defined.", - "name": "task", - "parameters": { - "properties": { - "agent": { - "description": "Declared subagent to use for the child agent", - "type": "string" - }, - "cwd": { - "description": "Working directory for the child agent. AGENTS.md and skills are discovered from here.", - "type": "string" - }, - "description": { - "description": "Short human-readable label for the delegated work", - "type": "string" - }, - "prompt": { - "description": "Focused instructions for the child agent", - "type": "string" - } - }, - "required": ["prompt"], - "type": "object" - }, - "strict": false, - "type": "function" - } - ] - } - }, - "headers": {}, - "method": "POST", - "url": "https://api.openai.com/v1/responses" - }, - "response": { - "body": { - "chunks": [ - "event: response.created\ndata: {\"type\":\"response.created\",\"response\":{\"id\":\"resp_054a16a2f2ea3217016a705b65dc0881a3a79d930ad7069d95\",\"object\":\"response\",\"created_at\":1785748325,\"status\":\"in_progress\",\"background\":false,\"completed_at\":null,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":null,\"max_tool_calls\":null,\"model\":\"gpt-5.4-nano-2026-03-17\",\"moderation\":null,\"output\":[],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":\"workflow:instrumentation:01KZ3E9TWTAEK4P4P3F63HKAKC::default::ta\",\"prompt_cache_retention\":\"24h\",\"reasoning\":{\"context\":\"current_turn\",\"effort\":\"none\",\"mode\":\"standard\",\"summary\":null},\"safety_identifier\":null,\"service_tier\":\"auto\",\"store\":false,\"temperature\":1,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tool_usage\":{\"image_gen\":{\"input_tokens\":0,\"input_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"output_tokens\":0,\"output_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"total_tokens\":0},\"web_search\":{\"num_requests\":0}},\"tools\":[{\"type\":\"function\",\"description\":\"Read a file or list a directory. For files, output is truncated to 2000 lines or 50KB — use offset/limit for large files. For directories, returns the list of entries.\",\"name\":\"read\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to read\"},\"offset\":{\"type\":\"number\",\"description\":\"Line number to start from (1-indexed)\"},\"limit\":{\"type\":\"number\",\"description\":\"Maximum number of lines to read\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Write content to a file. Creates the file and parent directories if they do not exist.\",\"name\":\"write\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"content\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to write\"},\"content\":{\"type\":\"string\",\"description\":\"Content to write to the file\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Edit a file using exact text replacement. The oldText must match a unique region of the file. Use replaceAll to replace all occurrences.\",\"name\":\"edit\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"oldText\",\"newText\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to edit\"},\"oldText\":{\"type\":\"string\",\"description\":\"Exact text to find (must be unique)\"},\"newText\":{\"type\":\"string\",\"description\":\"Replacement text\"},\"replaceAll\":{\"type\":\"boolean\",\"description\":\"Replace all occurrences\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Execute a bash command. Returns stdout and stderr. Output is truncated to the last 2000 lines or 50KB.\",\"name\":\"bash\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"command\"],\"properties\":{\"command\":{\"type\":\"string\",\"description\":\"Bash command to execute\"},\"timeout\":{\"type\":\"number\",\"description\":\"Timeout in seconds\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Search file contents for a regex pattern. Returns matching lines with file paths and line numbers.\",\"name\":\"grep\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Search pattern (regex)\"},\"path\":{\"type\":\"string\",\"description\":\"Directory or file to search (default: .)\"},\"include\":{\"type\":\"string\",\"description\":\"Glob filter, e.g. \\\"*.ts\\\"\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Find files by filename pattern using shell find -name semantics. Returns matching file paths.\",\"name\":\"glob\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Filename pattern, e.g. \\\"*.ts\\\"\"},\"path\":{\"type\":\"string\",\"description\":\"Directory to search in (default: .)\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Delegate a focused task to a detached child agent with its own context. Use this for independent research, file exploration, or parallel work. The task returns only its final answer to this conversation. No subagents are currently defined.\",\"name\":\"task\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"prompt\"],\"properties\":{\"description\":{\"type\":\"string\",\"description\":\"Short human-readable label for the delegated work\"},\"prompt\":{\"type\":\"string\",\"description\":\"Focused instructions for the child agent\"},\"agent\":{\"type\":\"string\",\"description\":\"Declared subagent to use for the child agent\"},\"cwd\":{\"type\":\"string\",\"description\":\"Working directory for the child agent. AGENTS.md and skills are discovered from here.\"}}},\"strict\":false}],\"top_logprobs\":0,\"top_p\":0.98,\"truncation\":\"disabled\",\"usage\":null,\"user\":null,\"metadata\":{}},\"sequence_number\":0}", - "event: response.in_progress\ndata: {\"type\":\"response.in_progress\",\"response\":{\"id\":\"resp_054a16a2f2ea3217016a705b65dc0881a3a79d930ad7069d95\",\"object\":\"response\",\"created_at\":1785748325,\"status\":\"in_progress\",\"background\":false,\"completed_at\":null,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":null,\"max_tool_calls\":null,\"model\":\"gpt-5.4-nano-2026-03-17\",\"moderation\":null,\"output\":[],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":\"workflow:instrumentation:01KZ3E9TWTAEK4P4P3F63HKAKC::default::ta\",\"prompt_cache_retention\":\"24h\",\"reasoning\":{\"context\":\"current_turn\",\"effort\":\"none\",\"mode\":\"standard\",\"summary\":null},\"safety_identifier\":null,\"service_tier\":\"auto\",\"store\":false,\"temperature\":1,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tool_usage\":{\"image_gen\":{\"input_tokens\":0,\"input_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"output_tokens\":0,\"output_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"total_tokens\":0},\"web_search\":{\"num_requests\":0}},\"tools\":[{\"type\":\"function\",\"description\":\"Read a file or list a directory. For files, output is truncated to 2000 lines or 50KB — use offset/limit for large files. For directories, returns the list of entries.\",\"name\":\"read\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to read\"},\"offset\":{\"type\":\"number\",\"description\":\"Line number to start from (1-indexed)\"},\"limit\":{\"type\":\"number\",\"description\":\"Maximum number of lines to read\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Write content to a file. Creates the file and parent directories if they do not exist.\",\"name\":\"write\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"content\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to write\"},\"content\":{\"type\":\"string\",\"description\":\"Content to write to the file\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Edit a file using exact text replacement. The oldText must match a unique region of the file. Use replaceAll to replace all occurrences.\",\"name\":\"edit\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"oldText\",\"newText\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to edit\"},\"oldText\":{\"type\":\"string\",\"description\":\"Exact text to find (must be unique)\"},\"newText\":{\"type\":\"string\",\"description\":\"Replacement text\"},\"replaceAll\":{\"type\":\"boolean\",\"description\":\"Replace all occurrences\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Execute a bash command. Returns stdout and stderr. Output is truncated to the last 2000 lines or 50KB.\",\"name\":\"bash\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"command\"],\"properties\":{\"command\":{\"type\":\"string\",\"description\":\"Bash command to execute\"},\"timeout\":{\"type\":\"number\",\"description\":\"Timeout in seconds\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Search file contents for a regex pattern. Returns matching lines with file paths and line numbers.\",\"name\":\"grep\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Search pattern (regex)\"},\"path\":{\"type\":\"string\",\"description\":\"Directory or file to search (default: .)\"},\"include\":{\"type\":\"string\",\"description\":\"Glob filter, e.g. \\\"*.ts\\\"\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Find files by filename pattern using shell find -name semantics. Returns matching file paths.\",\"name\":\"glob\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Filename pattern, e.g. \\\"*.ts\\\"\"},\"path\":{\"type\":\"string\",\"description\":\"Directory to search in (default: .)\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Delegate a focused task to a detached child agent with its own context. Use this for independent research, file exploration, or parallel work. The task returns only its final answer to this conversation. No subagents are currently defined.\",\"name\":\"task\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"prompt\"],\"properties\":{\"description\":{\"type\":\"string\",\"description\":\"Short human-readable label for the delegated work\"},\"prompt\":{\"type\":\"string\",\"description\":\"Focused instructions for the child agent\"},\"agent\":{\"type\":\"string\",\"description\":\"Declared subagent to use for the child agent\"},\"cwd\":{\"type\":\"string\",\"description\":\"Working directory for the child agent. AGENTS.md and skills are discovered from here.\"}}},\"strict\":false}],\"top_logprobs\":0,\"top_p\":0.98,\"truncation\":\"disabled\",\"usage\":null,\"user\":null,\"metadata\":{}},\"sequence_number\":1}", - "event: response.output_item.added\ndata: {\"type\":\"response.output_item.added\",\"item\":{\"id\":\"fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee\",\"type\":\"function_call\",\"status\":\"in_progress\",\"arguments\":\"\",\"call_id\":\"call_6zEBnJRmXFIkGFwNjWLZP4uC\",\"name\":\"bash\"},\"output_index\":0,\"sequence_number\":2}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"{\\\"\",\"item_id\":\"fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee\",\"obfuscation\":\"qYr2RE37LRYj8T\",\"output_index\":0,\"sequence_number\":3}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"command\",\"item_id\":\"fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee\",\"obfuscation\":\"5olycNCp4\",\"output_index\":0,\"sequence_number\":4}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"\\\":\\\"\",\"item_id\":\"fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee\",\"obfuscation\":\"TtIA15U6WTwyB\",\"output_index\":0,\"sequence_number\":5}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"ls\",\"item_id\":\"fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee\",\"obfuscation\":\"8gpmFqa6OBiMdp\",\"output_index\":0,\"sequence_number\":6}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" -\",\"item_id\":\"fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee\",\"obfuscation\":\"HqbfEOyA6n1XM5\",\"output_index\":0,\"sequence_number\":7}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"la\",\"item_id\":\"fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee\",\"obfuscation\":\"zkZ09ufXuvTuYH\",\"output_index\":0,\"sequence_number\":8}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" /\",\"item_id\":\"fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee\",\"obfuscation\":\"IsgyMPXjN3RyQT\",\"output_index\":0,\"sequence_number\":9}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"ver\",\"item_id\":\"fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee\",\"obfuscation\":\"QwfPCO1ymGdY7\",\"output_index\":0,\"sequence_number\":10}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"cel\",\"item_id\":\"fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee\",\"obfuscation\":\"ktEAC1neLBJD0\",\"output_index\":0,\"sequence_number\":11}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/s\",\"item_id\":\"fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee\",\"obfuscation\":\"qSxXdSs4TsmFt5\",\"output_index\":0,\"sequence_number\":12}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"andbox\",\"item_id\":\"fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee\",\"obfuscation\":\"NIrFRZbher\",\"output_index\":0,\"sequence_number\":13}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/\",\"item_id\":\"fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee\",\"obfuscation\":\"h1HKJqOxjX7neu9\",\"output_index\":0,\"sequence_number\":14}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"brain\",\"item_id\":\"fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee\",\"obfuscation\":\"g5de6pXeQeu\",\"output_index\":0,\"sequence_number\":15}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"trust\",\"item_id\":\"fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee\",\"obfuscation\":\"O8W7X6T6QJ9\",\"output_index\":0,\"sequence_number\":16}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"data\",\"item_id\":\"fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee\",\"obfuscation\":\"ZwdOUP0nNaaU\",\"output_index\":0,\"sequence_number\":17}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/\",\"item_id\":\"fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee\",\"obfuscation\":\"RdDtnAGnoUWZHMl\",\"output_index\":0,\"sequence_number\":18}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"brain\",\"item_id\":\"fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee\",\"obfuscation\":\"bJadnE3z1Pn\",\"output_index\":0,\"sequence_number\":19}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"trust\",\"item_id\":\"fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee\",\"obfuscation\":\"QdtHJab2Nju\",\"output_index\":0,\"sequence_number\":20}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"-sdk\",\"item_id\":\"fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee\",\"obfuscation\":\"d0BalsPUN30E\",\"output_index\":0,\"sequence_number\":21}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"-j\",\"item_id\":\"fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee\",\"obfuscation\":\"zkZSKvyjjnJqmd\",\"output_index\":0,\"sequence_number\":22}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"avascript\",\"item_id\":\"fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee\",\"obfuscation\":\"gidR8ss\",\"output_index\":0,\"sequence_number\":23}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/e\",\"item_id\":\"fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee\",\"obfuscation\":\"sTij2BeE8hhf4F\",\"output_index\":0,\"sequence_number\":24}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"2\",\"item_id\":\"fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee\",\"obfuscation\":\"U6tRlf8eVA0ZfFt\",\"output_index\":0,\"sequence_number\":25}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"e\",\"item_id\":\"fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee\",\"obfuscation\":\"3iym56BtyHXWWzj\",\"output_index\":0,\"sequence_number\":26}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/.\",\"item_id\":\"fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee\",\"obfuscation\":\"yPazRePg1CYCjy\",\"output_index\":0,\"sequence_number\":27}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"bt\",\"item_id\":\"fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee\",\"obfuscation\":\"XlYRyV5eP5RuAi\",\"output_index\":0,\"sequence_number\":28}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"-t\",\"item_id\":\"fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee\",\"obfuscation\":\"cEUjV7aq8IJmrd\",\"output_index\":0,\"sequence_number\":29}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"mp\",\"item_id\":\"fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee\",\"obfuscation\":\"llpLkANV2zyfnb\",\"output_index\":0,\"sequence_number\":30}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/run\",\"item_id\":\"fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee\",\"obfuscation\":\"flFQ8Wd13kY2\",\"output_index\":0,\"sequence_number\":31}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"-\",\"item_id\":\"fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee\",\"obfuscation\":\"6NV0XhXU74sXecB\",\"output_index\":0,\"sequence_number\":32}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"9\",\"item_id\":\"fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee\",\"obfuscation\":\"3e8DIPv3Q4zimTJ\",\"output_index\":0,\"sequence_number\":33}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"sp\",\"item_id\":\"fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee\",\"obfuscation\":\"CcYFgrShio2OGa\",\"output_index\":0,\"sequence_number\":34}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"u\",\"item_id\":\"fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee\",\"obfuscation\":\"SB8CftjcqKYqDAq\",\"output_index\":0,\"sequence_number\":35}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"Hg\",\"item_id\":\"fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee\",\"obfuscation\":\"o0TbOyqaOwnbjc\",\"output_index\":0,\"sequence_number\":36}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/sc\",\"item_id\":\"fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee\",\"obfuscation\":\"AxgExC2R2UrOj\",\"output_index\":0,\"sequence_number\":37}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"enarios\",\"item_id\":\"fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee\",\"obfuscation\":\"GktP5dj47\",\"output_index\":0,\"sequence_number\":38}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/fl\",\"item_id\":\"fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee\",\"obfuscation\":\"D8Qyuf0OZ5udR\",\"output_index\":0,\"sequence_number\":39}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"ue\",\"item_id\":\"fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee\",\"obfuscation\":\"rTVImQr26P3Vsq\",\"output_index\":0,\"sequence_number\":40}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"-in\",\"item_id\":\"fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee\",\"obfuscation\":\"SPXiRAIfqk20q\",\"output_index\":0,\"sequence_number\":41}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"strument\",\"item_id\":\"fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee\",\"obfuscation\":\"ukKgXlCi\",\"output_index\":0,\"sequence_number\":42}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"ation\",\"item_id\":\"fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee\",\"obfuscation\":\"q5sV1H9oEPR\",\"output_index\":0,\"sequence_number\":43}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/.\",\"item_id\":\"fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee\",\"obfuscation\":\"Vbdq3gYacR4ydn\",\"output_index\":0,\"sequence_number\":44}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"agents\",\"item_id\":\"fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee\",\"obfuscation\":\"iYjX4AJgNl\",\"output_index\":0,\"sequence_number\":45}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" &&\",\"item_id\":\"fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee\",\"obfuscation\":\"QpXEqJPwcIzx8\",\"output_index\":0,\"sequence_number\":46}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" find\",\"item_id\":\"fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee\",\"obfuscation\":\"idAm4VrdUd9\",\"output_index\":0,\"sequence_number\":47}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" /\",\"item_id\":\"fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee\",\"obfuscation\":\"xlH3WH5e7K7TVT\",\"output_index\":0,\"sequence_number\":48}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"ver\",\"item_id\":\"fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee\",\"obfuscation\":\"ci7N2JhvoR4v3\",\"output_index\":0,\"sequence_number\":49}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"cel\",\"item_id\":\"fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee\",\"obfuscation\":\"lNESXh61G4kcn\",\"output_index\":0,\"sequence_number\":50}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/s\",\"item_id\":\"fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee\",\"obfuscation\":\"JHLNRMdy9opWYh\",\"output_index\":0,\"sequence_number\":51}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"andbox\",\"item_id\":\"fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee\",\"obfuscation\":\"dgKik8Wq8j\",\"output_index\":0,\"sequence_number\":52}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/\",\"item_id\":\"fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee\",\"obfuscation\":\"ivSoP1Az4lunbMK\",\"output_index\":0,\"sequence_number\":53}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"brain\",\"item_id\":\"fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee\",\"obfuscation\":\"ei4Les3EoOx\",\"output_index\":0,\"sequence_number\":54}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"trust\",\"item_id\":\"fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee\",\"obfuscation\":\"j583CvShsMn\",\"output_index\":0,\"sequence_number\":55}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"data\",\"item_id\":\"fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee\",\"obfuscation\":\"8pCfiAeFxnGs\",\"output_index\":0,\"sequence_number\":56}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/\",\"item_id\":\"fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee\",\"obfuscation\":\"BrQSS7SwzKOQJMI\",\"output_index\":0,\"sequence_number\":57}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"brain\",\"item_id\":\"fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee\",\"obfuscation\":\"mq3EFmYB311\",\"output_index\":0,\"sequence_number\":58}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"trust\",\"item_id\":\"fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee\",\"obfuscation\":\"hO6jOpGPatJ\",\"output_index\":0,\"sequence_number\":59}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"-sdk\",\"item_id\":\"fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee\",\"obfuscation\":\"lbKGYWB3E1BM\",\"output_index\":0,\"sequence_number\":60}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"-j\",\"item_id\":\"fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee\",\"obfuscation\":\"u16CBhAKfJTH5L\",\"output_index\":0,\"sequence_number\":61}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"avascript\",\"item_id\":\"fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee\",\"obfuscation\":\"5hJroXc\",\"output_index\":0,\"sequence_number\":62}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/e\",\"item_id\":\"fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee\",\"obfuscation\":\"33pMuKaTNDmMqZ\",\"output_index\":0,\"sequence_number\":63}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"2\",\"item_id\":\"fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee\",\"obfuscation\":\"7PfD7iSXELGdvpH\",\"output_index\":0,\"sequence_number\":64}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"e\",\"item_id\":\"fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee\",\"obfuscation\":\"l89ZUwAGJ4COOHT\",\"output_index\":0,\"sequence_number\":65}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" -\",\"item_id\":\"fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee\",\"obfuscation\":\"heNzYKqh1OCQTD\",\"output_index\":0,\"sequence_number\":66}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"max\",\"item_id\":\"fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee\",\"obfuscation\":\"sU1Q9AhPAADW3\",\"output_index\":0,\"sequence_number\":67}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"depth\",\"item_id\":\"fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee\",\"obfuscation\":\"G2KErWebcGP\",\"output_index\":0,\"sequence_number\":68}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" \",\"item_id\":\"fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee\",\"obfuscation\":\"Hk3AzR5adCxbnFT\",\"output_index\":0,\"sequence_number\":69}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"4\",\"item_id\":\"fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee\",\"obfuscation\":\"2AwssIctaSpQjDt\",\"output_index\":0,\"sequence_number\":70}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" -\",\"item_id\":\"fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee\",\"obfuscation\":\"qTL2wxyksScrpT\",\"output_index\":0,\"sequence_number\":71}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"name\",\"item_id\":\"fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee\",\"obfuscation\":\"9dAiG7pNttSR\",\"output_index\":0,\"sequence_number\":72}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" AG\",\"item_id\":\"fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee\",\"obfuscation\":\"vyWvpkSniB10E\",\"output_index\":0,\"sequence_number\":73}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"ENTS\",\"item_id\":\"fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee\",\"obfuscation\":\"8X34qqxb7k4T\",\"output_index\":0,\"sequence_number\":74}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\".md\",\"item_id\":\"fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee\",\"obfuscation\":\"srlvyGZKNg8fy\",\"output_index\":0,\"sequence_number\":75}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" -\",\"item_id\":\"fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee\",\"obfuscation\":\"kX3C3ee4M4LAQZ\",\"output_index\":0,\"sequence_number\":76}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"print\",\"item_id\":\"fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee\",\"obfuscation\":\"kuuWd4dbfED\",\"output_index\":0,\"sequence_number\":77}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"\\\",\\\"\",\"item_id\":\"fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee\",\"obfuscation\":\"t1AZzeTRFs1dW\",\"output_index\":0,\"sequence_number\":78}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"timeout\",\"item_id\":\"fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee\",\"obfuscation\":\"J9nNvn2yP\",\"output_index\":0,\"sequence_number\":79}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"\\\":\",\"item_id\":\"fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee\",\"obfuscation\":\"HoGIKlERH8SubV\",\"output_index\":0,\"sequence_number\":80}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"100\",\"item_id\":\"fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee\",\"obfuscation\":\"K0vUuFSWy0H29\",\"output_index\":0,\"sequence_number\":81}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"000\",\"item_id\":\"fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee\",\"obfuscation\":\"iki9PdBtWpJFT\",\"output_index\":0,\"sequence_number\":82}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"}\",\"item_id\":\"fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee\",\"obfuscation\":\"XUhEnBL2ma1rfYy\",\"output_index\":0,\"sequence_number\":83}", - "event: response.function_call_arguments.done\ndata: {\"type\":\"response.function_call_arguments.done\",\"arguments\":\"{\\\"command\\\":\\\"find . -maxdepth 4 -name AGENTS.md -print\\\",\\\"timeout\\\":100000}\",\"item_id\":\"fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee\",\"output_index\":0,\"sequence_number\":84}", - "event: response.output_item.done\ndata: {\"type\":\"response.output_item.done\",\"item\":{\"id\":\"fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee\",\"type\":\"function_call\",\"status\":\"completed\",\"arguments\":\"{\\\"command\\\":\\\"find . -maxdepth 4 -name AGENTS.md -print\\\",\\\"timeout\\\":100000}\",\"call_id\":\"call_6zEBnJRmXFIkGFwNjWLZP4uC\",\"name\":\"bash\"},\"output_index\":0,\"sequence_number\":85}", - "event: response.completed\ndata: {\"type\":\"response.completed\",\"response\":{\"id\":\"resp_054a16a2f2ea3217016a705b65dc0881a3a79d930ad7069d95\",\"object\":\"response\",\"created_at\":1785748325,\"status\":\"completed\",\"background\":false,\"completed_at\":1785748326,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":null,\"max_tool_calls\":null,\"model\":\"gpt-5.4-nano-2026-03-17\",\"moderation\":null,\"output\":[{\"id\":\"fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee\",\"type\":\"function_call\",\"status\":\"completed\",\"arguments\":\"{\\\"command\\\":\\\"find . -maxdepth 4 -name AGENTS.md -print\\\",\\\"timeout\\\":100000}\",\"call_id\":\"call_6zEBnJRmXFIkGFwNjWLZP4uC\",\"name\":\"bash\"}],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":\"workflow:instrumentation:01KZ3E9TWTAEK4P4P3F63HKAKC::default::ta\",\"prompt_cache_retention\":\"24h\",\"reasoning\":{\"context\":\"current_turn\",\"effort\":\"none\",\"mode\":\"standard\",\"summary\":null},\"safety_identifier\":null,\"service_tier\":\"default\",\"store\":false,\"temperature\":1,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tool_usage\":{\"image_gen\":{\"input_tokens\":0,\"input_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"output_tokens\":0,\"output_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"total_tokens\":0},\"web_search\":{\"num_requests\":0}},\"tools\":[{\"type\":\"function\",\"description\":\"Read a file or list a directory. For files, output is truncated to 2000 lines or 50KB — use offset/limit for large files. For directories, returns the list of entries.\",\"name\":\"read\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to read\"},\"offset\":{\"type\":\"number\",\"description\":\"Line number to start from (1-indexed)\"},\"limit\":{\"type\":\"number\",\"description\":\"Maximum number of lines to read\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Write content to a file. Creates the file and parent directories if they do not exist.\",\"name\":\"write\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"content\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to write\"},\"content\":{\"type\":\"string\",\"description\":\"Content to write to the file\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Edit a file using exact text replacement. The oldText must match a unique region of the file. Use replaceAll to replace all occurrences.\",\"name\":\"edit\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"oldText\",\"newText\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to edit\"},\"oldText\":{\"type\":\"string\",\"description\":\"Exact text to find (must be unique)\"},\"newText\":{\"type\":\"string\",\"description\":\"Replacement text\"},\"replaceAll\":{\"type\":\"boolean\",\"description\":\"Replace all occurrences\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Execute a bash command. Returns stdout and stderr. Output is truncated to the last 2000 lines or 50KB.\",\"name\":\"bash\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"command\"],\"properties\":{\"command\":{\"type\":\"string\",\"description\":\"Bash command to execute\"},\"timeout\":{\"type\":\"number\",\"description\":\"Timeout in seconds\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Search file contents for a regex pattern. Returns matching lines with file paths and line numbers.\",\"name\":\"grep\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Search pattern (regex)\"},\"path\":{\"type\":\"string\",\"description\":\"Directory or file to search (default: .)\"},\"include\":{\"type\":\"string\",\"description\":\"Glob filter, e.g. \\\"*.ts\\\"\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Find files by filename pattern using shell find -name semantics. Returns matching file paths.\",\"name\":\"glob\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Filename pattern, e.g. \\\"*.ts\\\"\"},\"path\":{\"type\":\"string\",\"description\":\"Directory to search in (default: .)\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Delegate a focused task to a detached child agent with its own context. Use this for independent research, file exploration, or parallel work. The task returns only its final answer to this conversation. No subagents are currently defined.\",\"name\":\"task\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"prompt\"],\"properties\":{\"description\":{\"type\":\"string\",\"description\":\"Short human-readable label for the delegated work\"},\"prompt\":{\"type\":\"string\",\"description\":\"Focused instructions for the child agent\"},\"agent\":{\"type\":\"string\",\"description\":\"Declared subagent to use for the child agent\"},\"cwd\":{\"type\":\"string\",\"description\":\"Working directory for the child agent. AGENTS.md and skills are discovered from here.\"}}},\"strict\":false}],\"top_logprobs\":0,\"top_p\":0.98,\"truncation\":\"disabled\",\"usage\":{\"input_tokens\":1002,\"input_tokens_details\":{\"cache_write_tokens\":0,\"cached_tokens\":0},\"output_tokens\":94,\"output_tokens_details\":{\"reasoning_tokens\":0},\"total_tokens\":1096},\"user\":null,\"metadata\":{}},\"sequence_number\":86}" - ], - "kind": "sse" - }, - "headers": { - "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", - "alt-svc": "h3=\":443\"; ma=86400", - "cf-cache-status": "DYNAMIC", - "cf-ray": "a25432dc5a4c7bb6-IAD", - "connection": "keep-alive", - "content-type": "text/event-stream; charset=utf-8", - "date": "Mon, 03 Aug 2026 09:12:06 GMT", - "openai-organization": "braintrust-data", - "openai-processing-ms": "201", - "openai-project": "proj_vsCSXafhhByzWOThMrJcZiw9", - "openai-version": "2020-10-01", - "server": "cloudflare", - "set-cookie": "[REDACTED]", - "strict-transport-security": "max-age=31536000; includeSubDomains; preload", - "transfer-encoding": "chunked", - "x-content-type-options": "nosniff", - "x-ratelimit-limit-requests": "30000", - "x-ratelimit-limit-tokens": "180000000", - "x-ratelimit-remaining-requests": "29999", - "x-ratelimit-remaining-tokens": "179998545", - "x-ratelimit-reset-requests": "2ms", - "x-ratelimit-reset-tokens": "0s", - "x-request-id": "req_3ec73f40ef984f3ea2f7b4df76b2890e" - }, - "status": 200, - "statusText": "OK" - } - }, - { - "callIndex": 7, - "id": "bccc54d212e2503d", - "matchKey": "POST api.openai.com/v1/responses", - "recordedAt": "2026-08-03T09:12:08.056Z", - "request": { - "body": { - "kind": "json", - "value": { - "input": [ - { - "content": "You are running in headless mode with no human operator. Work autonomously — never ask questions, never wait for user input. Make your best judgment and proceed independently.\n\nYou are a deterministic Flue instrumentation test agent. Follow user instructions exactly. When asked for a marker, output only that marker and no extra text. When running a local skill file, read it yourself and do not delegate it to a task.\n\n## Available Skills\n\nThe following skills provide specialized instructions for specific tasks. When a task matches a skill description, activate that skill before proceeding so its full instructions are loaded. Skill instructions and supporting resources stay lazy until activation or explicit file reads.\n\n- **e2e-flue-skill** — Deterministic skill used by the Braintrust Flue instrumentation e2e test.\n\nDate: Mon, Aug 3, 2026\nWorking directory: .\n\nDirectory structure:\n.agents\n.flue\n.flue-build\n__cassettes__\n__snapshots__\nassertions.ts\ncassette-filter.mjs\nconstants.mjs\ndist\nnode_modules\npackage.json\npnpm-lock.yaml\nscenario.cli.mjs\nscenario.impl.mjs\nscenario.mjs\nscenario.test.ts\nscenario.ts\nscenario.v2.ts\nv2-assertions.ts\nversions", - "role": "developer" - }, - { - "content": [ - { - "text": "Run the skill named \"e2e-flue-skill\" with arguments {\"marker\":\"SKILL_DONE\"}. Output only the marker and no extra text when finished.", - "type": "input_text" - } - ], - "role": "user" - }, - { - "arguments": "{\"description\":\"Run e2e-flue-skill with marker SKILL_DONE\",\"prompt\":\"Run the skill named \\\"e2e-flue-skill\\\" with arguments {\\\"marker\\\":\\\"SKILL_DONE\\\"}. Ensure completion and report the final output marker only.\",\"agent\":\"e2e-flue-skill\",\"cwd\":\".\"}", - "call_id": "call_w6pyYZgnhlSOAXul75DAG1Cj", - "id": "fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6", - "name": "task", - "type": "function_call" - }, - { - "call_id": "call_w6pyYZgnhlSOAXul75DAG1Cj", - "output": "[flue] Subagent \"e2e-flue-skill\" is not declared. Available: (none).", - "type": "function_call_output" - }, - { - "arguments": "{\"command\":\"find . -maxdepth 4 -name AGENTS.md -print\",\"timeout\":100000}", - "call_id": "call_6zEBnJRmXFIkGFwNjWLZP4uC", - "id": "fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee", - "name": "bash", - "type": "function_call" - }, - { - "call_id": "call_6zEBnJRmXFIkGFwNjWLZP4uC", - "output": "total 4\ndrwxr-xr-x 3 vercel-sandbox vercel-sandbox 20 Aug 3 09:10 .\ndrwxr-xr-x 10 vercel-sandbox vercel-sandbox 4096 Aug 3 09:11 ..\ndrwxr-xr-x 3 vercel-sandbox vercel-sandbox 28 Aug 3 09:10 skills", - "type": "function_call_output" - } - ], - "model": "gpt-5.4-nano", - "prompt_cache_key": "[REDACTED]", - "reasoning": { - "effort": "none" - }, - "store": false, - "stream": true, - "tools": [ - { - "description": "Read a file or list a directory. For files, output is truncated to 2000 lines or 50KB — use offset/limit for large files. For directories, returns the list of entries.", - "name": "read", - "parameters": { - "properties": { - "limit": { - "description": "Maximum number of lines to read", - "type": "number" - }, - "offset": { - "description": "Line number to start from (1-indexed)", - "type": "number" - }, - "path": { - "description": "Path to the file to read", - "type": "string" - } - }, - "required": ["path"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Write content to a file. Creates the file and parent directories if they do not exist.", - "name": "write", - "parameters": { - "properties": { - "content": { - "description": "Content to write to the file", - "type": "string" - }, - "path": { - "description": "Path to the file to write", - "type": "string" - } - }, - "required": ["path", "content"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Edit a file using exact text replacement. The oldText must match a unique region of the file. Use replaceAll to replace all occurrences.", - "name": "edit", - "parameters": { - "properties": { - "newText": { - "description": "Replacement text", - "type": "string" - }, - "oldText": { - "description": "Exact text to find (must be unique)", - "type": "string" - }, - "path": { - "description": "Path to the file to edit", - "type": "string" - }, - "replaceAll": { - "description": "Replace all occurrences", - "type": "boolean" - } - }, - "required": ["path", "oldText", "newText"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Execute a bash command. Returns stdout and stderr. Output is truncated to the last 2000 lines or 50KB.", - "name": "bash", - "parameters": { - "properties": { - "command": { - "description": "Bash command to execute", - "type": "string" - }, - "timeout": { - "description": "Timeout in seconds", - "type": "number" - } - }, - "required": ["command"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Search file contents for a regex pattern. Returns matching lines with file paths and line numbers.", - "name": "grep", - "parameters": { - "properties": { - "include": { - "description": "Glob filter, e.g. \"*.ts\"", - "type": "string" - }, - "path": { - "description": "Directory or file to search (default: .)", - "type": "string" - }, - "pattern": { - "description": "Search pattern (regex)", - "type": "string" - } - }, - "required": ["pattern"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Find files by filename pattern using shell find -name semantics. Returns matching file paths.", - "name": "glob", - "parameters": { - "properties": { - "path": { - "description": "Directory to search in (default: .)", - "type": "string" - }, - "pattern": { - "description": "Filename pattern, e.g. \"*.ts\"", - "type": "string" - } - }, - "required": ["pattern"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Delegate a focused task to a detached child agent with its own context. Use this for independent research, file exploration, or parallel work. The task returns only its final answer to this conversation. No subagents are currently defined.", - "name": "task", - "parameters": { - "properties": { - "agent": { - "description": "Declared subagent to use for the child agent", - "type": "string" - }, - "cwd": { - "description": "Working directory for the child agent. AGENTS.md and skills are discovered from here.", - "type": "string" - }, - "description": { - "description": "Short human-readable label for the delegated work", - "type": "string" - }, - "prompt": { - "description": "Focused instructions for the child agent", - "type": "string" - } - }, - "required": ["prompt"], - "type": "object" - }, - "strict": false, - "type": "function" - } - ] - } - }, - "headers": {}, - "method": "POST", - "url": "https://api.openai.com/v1/responses" - }, - "response": { - "body": { - "chunks": [ - "event: response.created\ndata: {\"type\":\"response.created\",\"response\":{\"id\":\"resp_054a16a2f2ea3217016a705b66ec3081a3b53487cd28a912dd\",\"object\":\"response\",\"created_at\":1785748326,\"status\":\"in_progress\",\"background\":false,\"completed_at\":null,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":null,\"max_tool_calls\":null,\"model\":\"gpt-5.4-nano-2026-03-17\",\"moderation\":null,\"output\":[],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":\"workflow:instrumentation:01KZ3E9TWTAEK4P4P3F63HKAKC::default::ta\",\"prompt_cache_retention\":\"24h\",\"reasoning\":{\"context\":\"current_turn\",\"effort\":\"none\",\"mode\":\"standard\",\"summary\":null},\"safety_identifier\":null,\"service_tier\":\"auto\",\"store\":false,\"temperature\":1,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tool_usage\":{\"image_gen\":{\"input_tokens\":0,\"input_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"output_tokens\":0,\"output_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"total_tokens\":0},\"web_search\":{\"num_requests\":0}},\"tools\":[{\"type\":\"function\",\"description\":\"Read a file or list a directory. For files, output is truncated to 2000 lines or 50KB — use offset/limit for large files. For directories, returns the list of entries.\",\"name\":\"read\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to read\"},\"offset\":{\"type\":\"number\",\"description\":\"Line number to start from (1-indexed)\"},\"limit\":{\"type\":\"number\",\"description\":\"Maximum number of lines to read\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Write content to a file. Creates the file and parent directories if they do not exist.\",\"name\":\"write\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"content\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to write\"},\"content\":{\"type\":\"string\",\"description\":\"Content to write to the file\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Edit a file using exact text replacement. The oldText must match a unique region of the file. Use replaceAll to replace all occurrences.\",\"name\":\"edit\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"oldText\",\"newText\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to edit\"},\"oldText\":{\"type\":\"string\",\"description\":\"Exact text to find (must be unique)\"},\"newText\":{\"type\":\"string\",\"description\":\"Replacement text\"},\"replaceAll\":{\"type\":\"boolean\",\"description\":\"Replace all occurrences\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Execute a bash command. Returns stdout and stderr. Output is truncated to the last 2000 lines or 50KB.\",\"name\":\"bash\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"command\"],\"properties\":{\"command\":{\"type\":\"string\",\"description\":\"Bash command to execute\"},\"timeout\":{\"type\":\"number\",\"description\":\"Timeout in seconds\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Search file contents for a regex pattern. Returns matching lines with file paths and line numbers.\",\"name\":\"grep\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Search pattern (regex)\"},\"path\":{\"type\":\"string\",\"description\":\"Directory or file to search (default: .)\"},\"include\":{\"type\":\"string\",\"description\":\"Glob filter, e.g. \\\"*.ts\\\"\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Find files by filename pattern using shell find -name semantics. Returns matching file paths.\",\"name\":\"glob\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Filename pattern, e.g. \\\"*.ts\\\"\"},\"path\":{\"type\":\"string\",\"description\":\"Directory to search in (default: .)\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Delegate a focused task to a detached child agent with its own context. Use this for independent research, file exploration, or parallel work. The task returns only its final answer to this conversation. No subagents are currently defined.\",\"name\":\"task\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"prompt\"],\"properties\":{\"description\":{\"type\":\"string\",\"description\":\"Short human-readable label for the delegated work\"},\"prompt\":{\"type\":\"string\",\"description\":\"Focused instructions for the child agent\"},\"agent\":{\"type\":\"string\",\"description\":\"Declared subagent to use for the child agent\"},\"cwd\":{\"type\":\"string\",\"description\":\"Working directory for the child agent. AGENTS.md and skills are discovered from here.\"}}},\"strict\":false}],\"top_logprobs\":0,\"top_p\":0.98,\"truncation\":\"disabled\",\"usage\":null,\"user\":null,\"metadata\":{}},\"sequence_number\":0}", - "event: response.in_progress\ndata: {\"type\":\"response.in_progress\",\"response\":{\"id\":\"resp_054a16a2f2ea3217016a705b66ec3081a3b53487cd28a912dd\",\"object\":\"response\",\"created_at\":1785748326,\"status\":\"in_progress\",\"background\":false,\"completed_at\":null,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":null,\"max_tool_calls\":null,\"model\":\"gpt-5.4-nano-2026-03-17\",\"moderation\":null,\"output\":[],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":\"workflow:instrumentation:01KZ3E9TWTAEK4P4P3F63HKAKC::default::ta\",\"prompt_cache_retention\":\"24h\",\"reasoning\":{\"context\":\"current_turn\",\"effort\":\"none\",\"mode\":\"standard\",\"summary\":null},\"safety_identifier\":null,\"service_tier\":\"auto\",\"store\":false,\"temperature\":1,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tool_usage\":{\"image_gen\":{\"input_tokens\":0,\"input_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"output_tokens\":0,\"output_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"total_tokens\":0},\"web_search\":{\"num_requests\":0}},\"tools\":[{\"type\":\"function\",\"description\":\"Read a file or list a directory. For files, output is truncated to 2000 lines or 50KB — use offset/limit for large files. For directories, returns the list of entries.\",\"name\":\"read\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to read\"},\"offset\":{\"type\":\"number\",\"description\":\"Line number to start from (1-indexed)\"},\"limit\":{\"type\":\"number\",\"description\":\"Maximum number of lines to read\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Write content to a file. Creates the file and parent directories if they do not exist.\",\"name\":\"write\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"content\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to write\"},\"content\":{\"type\":\"string\",\"description\":\"Content to write to the file\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Edit a file using exact text replacement. The oldText must match a unique region of the file. Use replaceAll to replace all occurrences.\",\"name\":\"edit\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"oldText\",\"newText\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to edit\"},\"oldText\":{\"type\":\"string\",\"description\":\"Exact text to find (must be unique)\"},\"newText\":{\"type\":\"string\",\"description\":\"Replacement text\"},\"replaceAll\":{\"type\":\"boolean\",\"description\":\"Replace all occurrences\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Execute a bash command. Returns stdout and stderr. Output is truncated to the last 2000 lines or 50KB.\",\"name\":\"bash\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"command\"],\"properties\":{\"command\":{\"type\":\"string\",\"description\":\"Bash command to execute\"},\"timeout\":{\"type\":\"number\",\"description\":\"Timeout in seconds\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Search file contents for a regex pattern. Returns matching lines with file paths and line numbers.\",\"name\":\"grep\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Search pattern (regex)\"},\"path\":{\"type\":\"string\",\"description\":\"Directory or file to search (default: .)\"},\"include\":{\"type\":\"string\",\"description\":\"Glob filter, e.g. \\\"*.ts\\\"\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Find files by filename pattern using shell find -name semantics. Returns matching file paths.\",\"name\":\"glob\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Filename pattern, e.g. \\\"*.ts\\\"\"},\"path\":{\"type\":\"string\",\"description\":\"Directory to search in (default: .)\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Delegate a focused task to a detached child agent with its own context. Use this for independent research, file exploration, or parallel work. The task returns only its final answer to this conversation. No subagents are currently defined.\",\"name\":\"task\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"prompt\"],\"properties\":{\"description\":{\"type\":\"string\",\"description\":\"Short human-readable label for the delegated work\"},\"prompt\":{\"type\":\"string\",\"description\":\"Focused instructions for the child agent\"},\"agent\":{\"type\":\"string\",\"description\":\"Declared subagent to use for the child agent\"},\"cwd\":{\"type\":\"string\",\"description\":\"Working directory for the child agent. AGENTS.md and skills are discovered from here.\"}}},\"strict\":false}],\"top_logprobs\":0,\"top_p\":0.98,\"truncation\":\"disabled\",\"usage\":null,\"user\":null,\"metadata\":{}},\"sequence_number\":1}", - "event: response.output_item.added\ndata: {\"type\":\"response.output_item.added\",\"item\":{\"id\":\"fc_054a16a2f2ea3217016a705b67575881a3aacde02b9053a321\",\"type\":\"function_call\",\"status\":\"in_progress\",\"arguments\":\"\",\"call_id\":\"call_jQUCie0ZgaXUrZCDcezp0msl\",\"name\":\"bash\"},\"output_index\":0,\"sequence_number\":2}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"{\\\"\",\"item_id\":\"fc_054a16a2f2ea3217016a705b67575881a3aacde02b9053a321\",\"obfuscation\":\"5s5fPUr1lnx6LN\",\"output_index\":0,\"sequence_number\":3}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"command\",\"item_id\":\"fc_054a16a2f2ea3217016a705b67575881a3aacde02b9053a321\",\"obfuscation\":\"EzIXsl5P7\",\"output_index\":0,\"sequence_number\":4}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"\\\":\\\"\",\"item_id\":\"fc_054a16a2f2ea3217016a705b67575881a3aacde02b9053a321\",\"obfuscation\":\"rfpcd360vn0Zc\",\"output_index\":0,\"sequence_number\":5}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"find\",\"item_id\":\"fc_054a16a2f2ea3217016a705b67575881a3aacde02b9053a321\",\"obfuscation\":\"gbSugqPh322J\",\"output_index\":0,\"sequence_number\":6}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" /\",\"item_id\":\"fc_054a16a2f2ea3217016a705b67575881a3aacde02b9053a321\",\"obfuscation\":\"un8JuSPgzNW7Tq\",\"output_index\":0,\"sequence_number\":7}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"ver\",\"item_id\":\"fc_054a16a2f2ea3217016a705b67575881a3aacde02b9053a321\",\"obfuscation\":\"AT24QvJgyIKCb\",\"output_index\":0,\"sequence_number\":8}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"cel\",\"item_id\":\"fc_054a16a2f2ea3217016a705b67575881a3aacde02b9053a321\",\"obfuscation\":\"M2WqPhiDm0W0B\",\"output_index\":0,\"sequence_number\":9}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/s\",\"item_id\":\"fc_054a16a2f2ea3217016a705b67575881a3aacde02b9053a321\",\"obfuscation\":\"HnCl5JHASkkvRV\",\"output_index\":0,\"sequence_number\":10}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"andbox\",\"item_id\":\"fc_054a16a2f2ea3217016a705b67575881a3aacde02b9053a321\",\"obfuscation\":\"9hqImMoIYc\",\"output_index\":0,\"sequence_number\":11}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/\",\"item_id\":\"fc_054a16a2f2ea3217016a705b67575881a3aacde02b9053a321\",\"obfuscation\":\"G2MFkp35u1CeVM3\",\"output_index\":0,\"sequence_number\":12}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"brain\",\"item_id\":\"fc_054a16a2f2ea3217016a705b67575881a3aacde02b9053a321\",\"obfuscation\":\"S9DNif6AAaj\",\"output_index\":0,\"sequence_number\":13}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"trust\",\"item_id\":\"fc_054a16a2f2ea3217016a705b67575881a3aacde02b9053a321\",\"obfuscation\":\"KFRgE1qwn8O\",\"output_index\":0,\"sequence_number\":14}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"data\",\"item_id\":\"fc_054a16a2f2ea3217016a705b67575881a3aacde02b9053a321\",\"obfuscation\":\"iRZhcqXd9XRs\",\"output_index\":0,\"sequence_number\":15}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/\",\"item_id\":\"fc_054a16a2f2ea3217016a705b67575881a3aacde02b9053a321\",\"obfuscation\":\"9XF5P57pZo3uNAy\",\"output_index\":0,\"sequence_number\":16}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"brain\",\"item_id\":\"fc_054a16a2f2ea3217016a705b67575881a3aacde02b9053a321\",\"obfuscation\":\"4P9IWtruXv8\",\"output_index\":0,\"sequence_number\":17}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"trust\",\"item_id\":\"fc_054a16a2f2ea3217016a705b67575881a3aacde02b9053a321\",\"obfuscation\":\"6tLj1KQL0OS\",\"output_index\":0,\"sequence_number\":18}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"-sdk\",\"item_id\":\"fc_054a16a2f2ea3217016a705b67575881a3aacde02b9053a321\",\"obfuscation\":\"ZJX7xdMeL6wZ\",\"output_index\":0,\"sequence_number\":19}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"-j\",\"item_id\":\"fc_054a16a2f2ea3217016a705b67575881a3aacde02b9053a321\",\"obfuscation\":\"LJ7CGVyndyKtYf\",\"output_index\":0,\"sequence_number\":20}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"avascript\",\"item_id\":\"fc_054a16a2f2ea3217016a705b67575881a3aacde02b9053a321\",\"obfuscation\":\"xJcnaku\",\"output_index\":0,\"sequence_number\":21}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/e\",\"item_id\":\"fc_054a16a2f2ea3217016a705b67575881a3aacde02b9053a321\",\"obfuscation\":\"PfTOXv6id59uZk\",\"output_index\":0,\"sequence_number\":22}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"2\",\"item_id\":\"fc_054a16a2f2ea3217016a705b67575881a3aacde02b9053a321\",\"obfuscation\":\"ZygymPXfo8RsWR7\",\"output_index\":0,\"sequence_number\":23}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"e\",\"item_id\":\"fc_054a16a2f2ea3217016a705b67575881a3aacde02b9053a321\",\"obfuscation\":\"tcZjEPZR3q54QCO\",\"output_index\":0,\"sequence_number\":24}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/.\",\"item_id\":\"fc_054a16a2f2ea3217016a705b67575881a3aacde02b9053a321\",\"obfuscation\":\"Is3Ft8kklGHlSC\",\"output_index\":0,\"sequence_number\":25}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"bt\",\"item_id\":\"fc_054a16a2f2ea3217016a705b67575881a3aacde02b9053a321\",\"obfuscation\":\"J9UUPkfaryOCWA\",\"output_index\":0,\"sequence_number\":26}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"-t\",\"item_id\":\"fc_054a16a2f2ea3217016a705b67575881a3aacde02b9053a321\",\"obfuscation\":\"tUpNGsjI3owh2O\",\"output_index\":0,\"sequence_number\":27}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"mp\",\"item_id\":\"fc_054a16a2f2ea3217016a705b67575881a3aacde02b9053a321\",\"obfuscation\":\"uhvu4t3Z33cBrQ\",\"output_index\":0,\"sequence_number\":28}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/run\",\"item_id\":\"fc_054a16a2f2ea3217016a705b67575881a3aacde02b9053a321\",\"obfuscation\":\"EY7RAwkNVqHj\",\"output_index\":0,\"sequence_number\":29}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"-\",\"item_id\":\"fc_054a16a2f2ea3217016a705b67575881a3aacde02b9053a321\",\"obfuscation\":\"WzBJvCmbkvTjzRe\",\"output_index\":0,\"sequence_number\":30}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"9\",\"item_id\":\"fc_054a16a2f2ea3217016a705b67575881a3aacde02b9053a321\",\"obfuscation\":\"T1c8ikRVDyXs3K4\",\"output_index\":0,\"sequence_number\":31}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"sp\",\"item_id\":\"fc_054a16a2f2ea3217016a705b67575881a3aacde02b9053a321\",\"obfuscation\":\"TLa4CtuK49G4fL\",\"output_index\":0,\"sequence_number\":32}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"u\",\"item_id\":\"fc_054a16a2f2ea3217016a705b67575881a3aacde02b9053a321\",\"obfuscation\":\"vUKKjMsXw5sIkHv\",\"output_index\":0,\"sequence_number\":33}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"Hg\",\"item_id\":\"fc_054a16a2f2ea3217016a705b67575881a3aacde02b9053a321\",\"obfuscation\":\"SD4uFMMoVX66s1\",\"output_index\":0,\"sequence_number\":34}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/sc\",\"item_id\":\"fc_054a16a2f2ea3217016a705b67575881a3aacde02b9053a321\",\"obfuscation\":\"dsOEaf4EkLy14\",\"output_index\":0,\"sequence_number\":35}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"enarios\",\"item_id\":\"fc_054a16a2f2ea3217016a705b67575881a3aacde02b9053a321\",\"obfuscation\":\"Sk1iL5lxc\",\"output_index\":0,\"sequence_number\":36}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/fl\",\"item_id\":\"fc_054a16a2f2ea3217016a705b67575881a3aacde02b9053a321\",\"obfuscation\":\"j3F97pquP3wT0\",\"output_index\":0,\"sequence_number\":37}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"ue\",\"item_id\":\"fc_054a16a2f2ea3217016a705b67575881a3aacde02b9053a321\",\"obfuscation\":\"5SHqC6FW1engeD\",\"output_index\":0,\"sequence_number\":38}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"-in\",\"item_id\":\"fc_054a16a2f2ea3217016a705b67575881a3aacde02b9053a321\",\"obfuscation\":\"MKvWbeQhFJuyU\",\"output_index\":0,\"sequence_number\":39}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"strument\",\"item_id\":\"fc_054a16a2f2ea3217016a705b67575881a3aacde02b9053a321\",\"obfuscation\":\"0JPR573Y\",\"output_index\":0,\"sequence_number\":40}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"ation\",\"item_id\":\"fc_054a16a2f2ea3217016a705b67575881a3aacde02b9053a321\",\"obfuscation\":\"tJ0iHGcFnz2\",\"output_index\":0,\"sequence_number\":41}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" -\",\"item_id\":\"fc_054a16a2f2ea3217016a705b67575881a3aacde02b9053a321\",\"obfuscation\":\"2Xe2zCjtFpD1Aa\",\"output_index\":0,\"sequence_number\":42}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"max\",\"item_id\":\"fc_054a16a2f2ea3217016a705b67575881a3aacde02b9053a321\",\"obfuscation\":\"9fKJvO4eySsc7\",\"output_index\":0,\"sequence_number\":43}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"depth\",\"item_id\":\"fc_054a16a2f2ea3217016a705b67575881a3aacde02b9053a321\",\"obfuscation\":\"adTuj16Pog1\",\"output_index\":0,\"sequence_number\":44}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" \",\"item_id\":\"fc_054a16a2f2ea3217016a705b67575881a3aacde02b9053a321\",\"obfuscation\":\"RG1GXpp0dtqROaL\",\"output_index\":0,\"sequence_number\":45}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"4\",\"item_id\":\"fc_054a16a2f2ea3217016a705b67575881a3aacde02b9053a321\",\"obfuscation\":\"62nieO7lhOujIMV\",\"output_index\":0,\"sequence_number\":46}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" -\",\"item_id\":\"fc_054a16a2f2ea3217016a705b67575881a3aacde02b9053a321\",\"obfuscation\":\"f85mHnnTvDNVsm\",\"output_index\":0,\"sequence_number\":47}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"type\",\"item_id\":\"fc_054a16a2f2ea3217016a705b67575881a3aacde02b9053a321\",\"obfuscation\":\"oOkDwdNirXZn\",\"output_index\":0,\"sequence_number\":48}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" f\",\"item_id\":\"fc_054a16a2f2ea3217016a705b67575881a3aacde02b9053a321\",\"obfuscation\":\"7hsgm1tQLQmfWU\",\"output_index\":0,\"sequence_number\":49}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" -\",\"item_id\":\"fc_054a16a2f2ea3217016a705b67575881a3aacde02b9053a321\",\"obfuscation\":\"TEtJIF7zjvh5BC\",\"output_index\":0,\"sequence_number\":50}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"name\",\"item_id\":\"fc_054a16a2f2ea3217016a705b67575881a3aacde02b9053a321\",\"obfuscation\":\"TRWuvVnlxjuw\",\"output_index\":0,\"sequence_number\":51}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" AG\",\"item_id\":\"fc_054a16a2f2ea3217016a705b67575881a3aacde02b9053a321\",\"obfuscation\":\"OPMH46rRaY9wj\",\"output_index\":0,\"sequence_number\":52}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"ENTS\",\"item_id\":\"fc_054a16a2f2ea3217016a705b67575881a3aacde02b9053a321\",\"obfuscation\":\"Wx1ktH8noul5\",\"output_index\":0,\"sequence_number\":53}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\".md\",\"item_id\":\"fc_054a16a2f2ea3217016a705b67575881a3aacde02b9053a321\",\"obfuscation\":\"Oo8KevjbHLtSH\",\"output_index\":0,\"sequence_number\":54}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" -\",\"item_id\":\"fc_054a16a2f2ea3217016a705b67575881a3aacde02b9053a321\",\"obfuscation\":\"W1tKVt07sDFz6L\",\"output_index\":0,\"sequence_number\":55}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"print\",\"item_id\":\"fc_054a16a2f2ea3217016a705b67575881a3aacde02b9053a321\",\"obfuscation\":\"Ca0wTzfowmG\",\"output_index\":0,\"sequence_number\":56}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" -\",\"item_id\":\"fc_054a16a2f2ea3217016a705b67575881a3aacde02b9053a321\",\"obfuscation\":\"InAeZlZgCo2EWg\",\"output_index\":0,\"sequence_number\":57}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"exec\",\"item_id\":\"fc_054a16a2f2ea3217016a705b67575881a3aacde02b9053a321\",\"obfuscation\":\"xuVbI7qC8ZR7\",\"output_index\":0,\"sequence_number\":58}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" sed\",\"item_id\":\"fc_054a16a2f2ea3217016a705b67575881a3aacde02b9053a321\",\"obfuscation\":\"rWSvxjZ31UQg\",\"output_index\":0,\"sequence_number\":59}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" -\",\"item_id\":\"fc_054a16a2f2ea3217016a705b67575881a3aacde02b9053a321\",\"obfuscation\":\"LsdCqDwIUZ1aXY\",\"output_index\":0,\"sequence_number\":60}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"n\",\"item_id\":\"fc_054a16a2f2ea3217016a705b67575881a3aacde02b9053a321\",\"obfuscation\":\"9t5AteQjutiyqe3\",\"output_index\":0,\"sequence_number\":61}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" '\",\"item_id\":\"fc_054a16a2f2ea3217016a705b67575881a3aacde02b9053a321\",\"obfuscation\":\"b8zSZflAk9EXLf\",\"output_index\":0,\"sequence_number\":62}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"1\",\"item_id\":\"fc_054a16a2f2ea3217016a705b67575881a3aacde02b9053a321\",\"obfuscation\":\"i7Uvk2jgzfKhd4j\",\"output_index\":0,\"sequence_number\":63}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\",\",\"item_id\":\"fc_054a16a2f2ea3217016a705b67575881a3aacde02b9053a321\",\"obfuscation\":\"dY2J1TUAF8vRFo8\",\"output_index\":0,\"sequence_number\":64}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"200\",\"item_id\":\"fc_054a16a2f2ea3217016a705b67575881a3aacde02b9053a321\",\"obfuscation\":\"3t3yMCd2EF2u3\",\"output_index\":0,\"sequence_number\":65}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"p\",\"item_id\":\"fc_054a16a2f2ea3217016a705b67575881a3aacde02b9053a321\",\"obfuscation\":\"LI717GeLz7d3rCb\",\"output_index\":0,\"sequence_number\":66}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"'\",\"item_id\":\"fc_054a16a2f2ea3217016a705b67575881a3aacde02b9053a321\",\"obfuscation\":\"aUymdN14yxm7RVU\",\"output_index\":0,\"sequence_number\":67}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" {}\",\"item_id\":\"fc_054a16a2f2ea3217016a705b67575881a3aacde02b9053a321\",\"obfuscation\":\"SsZA4kHNYtJqo\",\"output_index\":0,\"sequence_number\":68}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" \\\\\\\\\",\"item_id\":\"fc_054a16a2f2ea3217016a705b67575881a3aacde02b9053a321\",\"obfuscation\":\"VZIldvECynY3p\",\"output_index\":0,\"sequence_number\":69}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\";\",\"item_id\":\"fc_054a16a2f2ea3217016a705b67575881a3aacde02b9053a321\",\"obfuscation\":\"LXiCXRoQhZXP5HK\",\"output_index\":0,\"sequence_number\":70}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"\\\",\\\"\",\"item_id\":\"fc_054a16a2f2ea3217016a705b67575881a3aacde02b9053a321\",\"obfuscation\":\"lN4m4dNvIh4ID\",\"output_index\":0,\"sequence_number\":71}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"timeout\",\"item_id\":\"fc_054a16a2f2ea3217016a705b67575881a3aacde02b9053a321\",\"obfuscation\":\"bGxGZD252\",\"output_index\":0,\"sequence_number\":72}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"\\\":\",\"item_id\":\"fc_054a16a2f2ea3217016a705b67575881a3aacde02b9053a321\",\"obfuscation\":\"Zz2e8ECX6n6SVc\",\"output_index\":0,\"sequence_number\":73}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"100\",\"item_id\":\"fc_054a16a2f2ea3217016a705b67575881a3aacde02b9053a321\",\"obfuscation\":\"Zi6V6iOPLR6s7\",\"output_index\":0,\"sequence_number\":74}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"000\",\"item_id\":\"fc_054a16a2f2ea3217016a705b67575881a3aacde02b9053a321\",\"obfuscation\":\"rp63i4YuRxltd\",\"output_index\":0,\"sequence_number\":75}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"}\",\"item_id\":\"fc_054a16a2f2ea3217016a705b67575881a3aacde02b9053a321\",\"obfuscation\":\"74rLY4yKHw7IjTn\",\"output_index\":0,\"sequence_number\":76}", - "event: response.function_call_arguments.done\ndata: {\"type\":\"response.function_call_arguments.done\",\"arguments\":\"{\\\"command\\\":\\\"find . -maxdepth 4 -type f -name AGENTS.md -print -exec sed -n '1,200p' {} \\\\\\\\;\\\",\\\"timeout\\\":100000}\",\"item_id\":\"fc_054a16a2f2ea3217016a705b67575881a3aacde02b9053a321\",\"output_index\":0,\"sequence_number\":77}", - "event: response.output_item.done\ndata: {\"type\":\"response.output_item.done\",\"item\":{\"id\":\"fc_054a16a2f2ea3217016a705b67575881a3aacde02b9053a321\",\"type\":\"function_call\",\"status\":\"completed\",\"arguments\":\"{\\\"command\\\":\\\"find . -maxdepth 4 -type f -name AGENTS.md -print -exec sed -n '1,200p' {} \\\\\\\\;\\\",\\\"timeout\\\":100000}\",\"call_id\":\"call_jQUCie0ZgaXUrZCDcezp0msl\",\"name\":\"bash\"},\"output_index\":0,\"sequence_number\":78}", - "event: response.completed\ndata: {\"type\":\"response.completed\",\"response\":{\"id\":\"resp_054a16a2f2ea3217016a705b66ec3081a3b53487cd28a912dd\",\"object\":\"response\",\"created_at\":1785748326,\"status\":\"completed\",\"background\":false,\"completed_at\":1785748328,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":null,\"max_tool_calls\":null,\"model\":\"gpt-5.4-nano-2026-03-17\",\"moderation\":null,\"output\":[{\"id\":\"fc_054a16a2f2ea3217016a705b67575881a3aacde02b9053a321\",\"type\":\"function_call\",\"status\":\"completed\",\"arguments\":\"{\\\"command\\\":\\\"find . -maxdepth 4 -type f -name AGENTS.md -print -exec sed -n '1,200p' {} \\\\\\\\;\\\",\\\"timeout\\\":100000}\",\"call_id\":\"call_jQUCie0ZgaXUrZCDcezp0msl\",\"name\":\"bash\"}],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":\"workflow:instrumentation:01KZ3E9TWTAEK4P4P3F63HKAKC::default::ta\",\"prompt_cache_retention\":\"24h\",\"reasoning\":{\"context\":\"current_turn\",\"effort\":\"none\",\"mode\":\"standard\",\"summary\":null},\"safety_identifier\":null,\"service_tier\":\"default\",\"store\":false,\"temperature\":1,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tool_usage\":{\"image_gen\":{\"input_tokens\":0,\"input_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"output_tokens\":0,\"output_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"total_tokens\":0},\"web_search\":{\"num_requests\":0}},\"tools\":[{\"type\":\"function\",\"description\":\"Read a file or list a directory. For files, output is truncated to 2000 lines or 50KB — use offset/limit for large files. For directories, returns the list of entries.\",\"name\":\"read\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to read\"},\"offset\":{\"type\":\"number\",\"description\":\"Line number to start from (1-indexed)\"},\"limit\":{\"type\":\"number\",\"description\":\"Maximum number of lines to read\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Write content to a file. Creates the file and parent directories if they do not exist.\",\"name\":\"write\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"content\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to write\"},\"content\":{\"type\":\"string\",\"description\":\"Content to write to the file\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Edit a file using exact text replacement. The oldText must match a unique region of the file. Use replaceAll to replace all occurrences.\",\"name\":\"edit\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"oldText\",\"newText\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to edit\"},\"oldText\":{\"type\":\"string\",\"description\":\"Exact text to find (must be unique)\"},\"newText\":{\"type\":\"string\",\"description\":\"Replacement text\"},\"replaceAll\":{\"type\":\"boolean\",\"description\":\"Replace all occurrences\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Execute a bash command. Returns stdout and stderr. Output is truncated to the last 2000 lines or 50KB.\",\"name\":\"bash\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"command\"],\"properties\":{\"command\":{\"type\":\"string\",\"description\":\"Bash command to execute\"},\"timeout\":{\"type\":\"number\",\"description\":\"Timeout in seconds\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Search file contents for a regex pattern. Returns matching lines with file paths and line numbers.\",\"name\":\"grep\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Search pattern (regex)\"},\"path\":{\"type\":\"string\",\"description\":\"Directory or file to search (default: .)\"},\"include\":{\"type\":\"string\",\"description\":\"Glob filter, e.g. \\\"*.ts\\\"\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Find files by filename pattern using shell find -name semantics. Returns matching file paths.\",\"name\":\"glob\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Filename pattern, e.g. \\\"*.ts\\\"\"},\"path\":{\"type\":\"string\",\"description\":\"Directory to search in (default: .)\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Delegate a focused task to a detached child agent with its own context. Use this for independent research, file exploration, or parallel work. The task returns only its final answer to this conversation. No subagents are currently defined.\",\"name\":\"task\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"prompt\"],\"properties\":{\"description\":{\"type\":\"string\",\"description\":\"Short human-readable label for the delegated work\"},\"prompt\":{\"type\":\"string\",\"description\":\"Focused instructions for the child agent\"},\"agent\":{\"type\":\"string\",\"description\":\"Declared subagent to use for the child agent\"},\"cwd\":{\"type\":\"string\",\"description\":\"Working directory for the child agent. AGENTS.md and skills are discovered from here.\"}}},\"strict\":false}],\"top_logprobs\":0,\"top_p\":0.98,\"truncation\":\"disabled\",\"usage\":{\"input_tokens\":1197,\"input_tokens_details\":{\"cache_write_tokens\":0,\"cached_tokens\":0},\"output_tokens\":87,\"output_tokens_details\":{\"reasoning_tokens\":0},\"total_tokens\":1284},\"user\":null,\"metadata\":{}},\"sequence_number\":79}" - ], - "kind": "sse" - }, - "headers": { - "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", - "alt-svc": "h3=\":443\"; ma=86400", - "cf-cache-status": "DYNAMIC", - "cf-ray": "a25432e308c57bb6-IAD", - "connection": "keep-alive", - "content-type": "text/event-stream; charset=utf-8", - "date": "Mon, 03 Aug 2026 09:12:07 GMT", - "openai-organization": "braintrust-data", - "openai-processing-ms": "181", - "openai-project": "proj_vsCSXafhhByzWOThMrJcZiw9", - "openai-version": "2020-10-01", - "server": "cloudflare", - "set-cookie": "[REDACTED]", - "strict-transport-security": "max-age=31536000; includeSubDomains; preload", - "transfer-encoding": "chunked", - "x-content-type-options": "nosniff", - "x-ratelimit-limit-requests": "30000", - "x-ratelimit-limit-tokens": "180000000", - "x-ratelimit-remaining-requests": "29999", - "x-ratelimit-remaining-tokens": "179998347", - "x-ratelimit-reset-requests": "2ms", - "x-ratelimit-reset-tokens": "0s", - "x-request-id": "req_5c4fd875329344cfa3d20c5925805429" - }, - "status": 200, - "statusText": "OK" - } - }, - { - "callIndex": 8, - "id": "8eec740c1be01c5c", - "matchKey": "POST api.openai.com/v1/responses", - "recordedAt": "2026-08-03T09:12:09.317Z", - "request": { - "body": { - "kind": "json", - "value": { - "input": [ - { - "content": "You are running in headless mode with no human operator. Work autonomously — never ask questions, never wait for user input. Make your best judgment and proceed independently.\n\nYou are a deterministic Flue instrumentation test agent. Follow user instructions exactly. When asked for a marker, output only that marker and no extra text. When running a local skill file, read it yourself and do not delegate it to a task.\n\n## Available Skills\n\nThe following skills provide specialized instructions for specific tasks. When a task matches a skill description, activate that skill before proceeding so its full instructions are loaded. Skill instructions and supporting resources stay lazy until activation or explicit file reads.\n\n- **e2e-flue-skill** — Deterministic skill used by the Braintrust Flue instrumentation e2e test.\n\nDate: Mon, Aug 3, 2026\nWorking directory: .\n\nDirectory structure:\n.agents\n.flue\n.flue-build\n__cassettes__\n__snapshots__\nassertions.ts\ncassette-filter.mjs\nconstants.mjs\ndist\nnode_modules\npackage.json\npnpm-lock.yaml\nscenario.cli.mjs\nscenario.impl.mjs\nscenario.mjs\nscenario.test.ts\nscenario.ts\nscenario.v2.ts\nv2-assertions.ts\nversions", - "role": "developer" - }, - { - "content": [ - { - "text": "Run the skill named \"e2e-flue-skill\" with arguments {\"marker\":\"SKILL_DONE\"}. Output only the marker and no extra text when finished.", - "type": "input_text" - } - ], - "role": "user" - }, - { - "arguments": "{\"description\":\"Run e2e-flue-skill with marker SKILL_DONE\",\"prompt\":\"Run the skill named \\\"e2e-flue-skill\\\" with arguments {\\\"marker\\\":\\\"SKILL_DONE\\\"}. Ensure completion and report the final output marker only.\",\"agent\":\"e2e-flue-skill\",\"cwd\":\".\"}", - "call_id": "call_w6pyYZgnhlSOAXul75DAG1Cj", - "id": "fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6", - "name": "task", - "type": "function_call" - }, - { - "call_id": "call_w6pyYZgnhlSOAXul75DAG1Cj", - "output": "[flue] Subagent \"e2e-flue-skill\" is not declared. Available: (none).", - "type": "function_call_output" - }, - { - "arguments": "{\"command\":\"find . -maxdepth 4 -name AGENTS.md -print\",\"timeout\":100000}", - "call_id": "call_6zEBnJRmXFIkGFwNjWLZP4uC", - "id": "fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee", - "name": "bash", - "type": "function_call" - }, - { - "call_id": "call_6zEBnJRmXFIkGFwNjWLZP4uC", - "output": "total 4\ndrwxr-xr-x 3 vercel-sandbox vercel-sandbox 20 Aug 3 09:10 .\ndrwxr-xr-x 10 vercel-sandbox vercel-sandbox 4096 Aug 3 09:11 ..\ndrwxr-xr-x 3 vercel-sandbox vercel-sandbox 28 Aug 3 09:10 skills", - "type": "function_call_output" - }, - { - "arguments": "{\"command\":\"find . -maxdepth 4 -type f -name AGENTS.md -print -exec sed -n '1,200p' {} \\\\;\",\"timeout\":100000}", - "call_id": "call_jQUCie0ZgaXUrZCDcezp0msl", - "id": "fc_054a16a2f2ea3217016a705b67575881a3aacde02b9053a321", - "name": "bash", - "type": "function_call" - }, - { - "call_id": "call_jQUCie0ZgaXUrZCDcezp0msl", - "output": "(no output)", - "type": "function_call_output" - } - ], - "model": "gpt-5.4-nano", - "prompt_cache_key": "[REDACTED]", - "reasoning": { - "effort": "none" - }, - "store": false, - "stream": true, - "tools": [ - { - "description": "Read a file or list a directory. For files, output is truncated to 2000 lines or 50KB — use offset/limit for large files. For directories, returns the list of entries.", - "name": "read", - "parameters": { - "properties": { - "limit": { - "description": "Maximum number of lines to read", - "type": "number" - }, - "offset": { - "description": "Line number to start from (1-indexed)", - "type": "number" - }, - "path": { - "description": "Path to the file to read", - "type": "string" - } - }, - "required": ["path"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Write content to a file. Creates the file and parent directories if they do not exist.", - "name": "write", - "parameters": { - "properties": { - "content": { - "description": "Content to write to the file", - "type": "string" - }, - "path": { - "description": "Path to the file to write", - "type": "string" - } - }, - "required": ["path", "content"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Edit a file using exact text replacement. The oldText must match a unique region of the file. Use replaceAll to replace all occurrences.", - "name": "edit", - "parameters": { - "properties": { - "newText": { - "description": "Replacement text", - "type": "string" - }, - "oldText": { - "description": "Exact text to find (must be unique)", - "type": "string" - }, - "path": { - "description": "Path to the file to edit", - "type": "string" - }, - "replaceAll": { - "description": "Replace all occurrences", - "type": "boolean" - } - }, - "required": ["path", "oldText", "newText"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Execute a bash command. Returns stdout and stderr. Output is truncated to the last 2000 lines or 50KB.", - "name": "bash", - "parameters": { - "properties": { - "command": { - "description": "Bash command to execute", - "type": "string" - }, - "timeout": { - "description": "Timeout in seconds", - "type": "number" - } - }, - "required": ["command"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Search file contents for a regex pattern. Returns matching lines with file paths and line numbers.", - "name": "grep", - "parameters": { - "properties": { - "include": { - "description": "Glob filter, e.g. \"*.ts\"", - "type": "string" - }, - "path": { - "description": "Directory or file to search (default: .)", - "type": "string" - }, - "pattern": { - "description": "Search pattern (regex)", - "type": "string" - } - }, - "required": ["pattern"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Find files by filename pattern using shell find -name semantics. Returns matching file paths.", - "name": "glob", - "parameters": { - "properties": { - "path": { - "description": "Directory to search in (default: .)", - "type": "string" - }, - "pattern": { - "description": "Filename pattern, e.g. \"*.ts\"", - "type": "string" - } - }, - "required": ["pattern"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Delegate a focused task to a detached child agent with its own context. Use this for independent research, file exploration, or parallel work. The task returns only its final answer to this conversation. No subagents are currently defined.", - "name": "task", - "parameters": { - "properties": { - "agent": { - "description": "Declared subagent to use for the child agent", - "type": "string" - }, - "cwd": { - "description": "Working directory for the child agent. AGENTS.md and skills are discovered from here.", - "type": "string" - }, - "description": { - "description": "Short human-readable label for the delegated work", - "type": "string" - }, - "prompt": { - "description": "Focused instructions for the child agent", - "type": "string" - } - }, - "required": ["prompt"], - "type": "object" - }, - "strict": false, - "type": "function" - } - ] - } - }, - "headers": {}, - "method": "POST", - "url": "https://api.openai.com/v1/responses" - }, - "response": { - "body": { - "chunks": [ - "event: response.created\ndata: {\"type\":\"response.created\",\"response\":{\"id\":\"resp_054a16a2f2ea3217016a705b6820c481a3b64f6527345dc3ca\",\"object\":\"response\",\"created_at\":1785748328,\"status\":\"in_progress\",\"background\":false,\"completed_at\":null,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":null,\"max_tool_calls\":null,\"model\":\"gpt-5.4-nano-2026-03-17\",\"moderation\":null,\"output\":[],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":\"workflow:instrumentation:01KZ3E9TWTAEK4P4P3F63HKAKC::default::ta\",\"prompt_cache_retention\":\"24h\",\"reasoning\":{\"context\":\"current_turn\",\"effort\":\"none\",\"mode\":\"standard\",\"summary\":null},\"safety_identifier\":null,\"service_tier\":\"auto\",\"store\":false,\"temperature\":1,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tool_usage\":{\"image_gen\":{\"input_tokens\":0,\"input_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"output_tokens\":0,\"output_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"total_tokens\":0},\"web_search\":{\"num_requests\":0}},\"tools\":[{\"type\":\"function\",\"description\":\"Read a file or list a directory. For files, output is truncated to 2000 lines or 50KB — use offset/limit for large files. For directories, returns the list of entries.\",\"name\":\"read\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to read\"},\"offset\":{\"type\":\"number\",\"description\":\"Line number to start from (1-indexed)\"},\"limit\":{\"type\":\"number\",\"description\":\"Maximum number of lines to read\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Write content to a file. Creates the file and parent directories if they do not exist.\",\"name\":\"write\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"content\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to write\"},\"content\":{\"type\":\"string\",\"description\":\"Content to write to the file\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Edit a file using exact text replacement. The oldText must match a unique region of the file. Use replaceAll to replace all occurrences.\",\"name\":\"edit\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"oldText\",\"newText\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to edit\"},\"oldText\":{\"type\":\"string\",\"description\":\"Exact text to find (must be unique)\"},\"newText\":{\"type\":\"string\",\"description\":\"Replacement text\"},\"replaceAll\":{\"type\":\"boolean\",\"description\":\"Replace all occurrences\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Execute a bash command. Returns stdout and stderr. Output is truncated to the last 2000 lines or 50KB.\",\"name\":\"bash\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"command\"],\"properties\":{\"command\":{\"type\":\"string\",\"description\":\"Bash command to execute\"},\"timeout\":{\"type\":\"number\",\"description\":\"Timeout in seconds\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Search file contents for a regex pattern. Returns matching lines with file paths and line numbers.\",\"name\":\"grep\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Search pattern (regex)\"},\"path\":{\"type\":\"string\",\"description\":\"Directory or file to search (default: .)\"},\"include\":{\"type\":\"string\",\"description\":\"Glob filter, e.g. \\\"*.ts\\\"\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Find files by filename pattern using shell find -name semantics. Returns matching file paths.\",\"name\":\"glob\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Filename pattern, e.g. \\\"*.ts\\\"\"},\"path\":{\"type\":\"string\",\"description\":\"Directory to search in (default: .)\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Delegate a focused task to a detached child agent with its own context. Use this for independent research, file exploration, or parallel work. The task returns only its final answer to this conversation. No subagents are currently defined.\",\"name\":\"task\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"prompt\"],\"properties\":{\"description\":{\"type\":\"string\",\"description\":\"Short human-readable label for the delegated work\"},\"prompt\":{\"type\":\"string\",\"description\":\"Focused instructions for the child agent\"},\"agent\":{\"type\":\"string\",\"description\":\"Declared subagent to use for the child agent\"},\"cwd\":{\"type\":\"string\",\"description\":\"Working directory for the child agent. AGENTS.md and skills are discovered from here.\"}}},\"strict\":false}],\"top_logprobs\":0,\"top_p\":0.98,\"truncation\":\"disabled\",\"usage\":null,\"user\":null,\"metadata\":{}},\"sequence_number\":0}", - "event: response.in_progress\ndata: {\"type\":\"response.in_progress\",\"response\":{\"id\":\"resp_054a16a2f2ea3217016a705b6820c481a3b64f6527345dc3ca\",\"object\":\"response\",\"created_at\":1785748328,\"status\":\"in_progress\",\"background\":false,\"completed_at\":null,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":null,\"max_tool_calls\":null,\"model\":\"gpt-5.4-nano-2026-03-17\",\"moderation\":null,\"output\":[],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":\"workflow:instrumentation:01KZ3E9TWTAEK4P4P3F63HKAKC::default::ta\",\"prompt_cache_retention\":\"24h\",\"reasoning\":{\"context\":\"current_turn\",\"effort\":\"none\",\"mode\":\"standard\",\"summary\":null},\"safety_identifier\":null,\"service_tier\":\"auto\",\"store\":false,\"temperature\":1,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tool_usage\":{\"image_gen\":{\"input_tokens\":0,\"input_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"output_tokens\":0,\"output_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"total_tokens\":0},\"web_search\":{\"num_requests\":0}},\"tools\":[{\"type\":\"function\",\"description\":\"Read a file or list a directory. For files, output is truncated to 2000 lines or 50KB — use offset/limit for large files. For directories, returns the list of entries.\",\"name\":\"read\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to read\"},\"offset\":{\"type\":\"number\",\"description\":\"Line number to start from (1-indexed)\"},\"limit\":{\"type\":\"number\",\"description\":\"Maximum number of lines to read\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Write content to a file. Creates the file and parent directories if they do not exist.\",\"name\":\"write\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"content\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to write\"},\"content\":{\"type\":\"string\",\"description\":\"Content to write to the file\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Edit a file using exact text replacement. The oldText must match a unique region of the file. Use replaceAll to replace all occurrences.\",\"name\":\"edit\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"oldText\",\"newText\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to edit\"},\"oldText\":{\"type\":\"string\",\"description\":\"Exact text to find (must be unique)\"},\"newText\":{\"type\":\"string\",\"description\":\"Replacement text\"},\"replaceAll\":{\"type\":\"boolean\",\"description\":\"Replace all occurrences\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Execute a bash command. Returns stdout and stderr. Output is truncated to the last 2000 lines or 50KB.\",\"name\":\"bash\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"command\"],\"properties\":{\"command\":{\"type\":\"string\",\"description\":\"Bash command to execute\"},\"timeout\":{\"type\":\"number\",\"description\":\"Timeout in seconds\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Search file contents for a regex pattern. Returns matching lines with file paths and line numbers.\",\"name\":\"grep\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Search pattern (regex)\"},\"path\":{\"type\":\"string\",\"description\":\"Directory or file to search (default: .)\"},\"include\":{\"type\":\"string\",\"description\":\"Glob filter, e.g. \\\"*.ts\\\"\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Find files by filename pattern using shell find -name semantics. Returns matching file paths.\",\"name\":\"glob\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Filename pattern, e.g. \\\"*.ts\\\"\"},\"path\":{\"type\":\"string\",\"description\":\"Directory to search in (default: .)\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Delegate a focused task to a detached child agent with its own context. Use this for independent research, file exploration, or parallel work. The task returns only its final answer to this conversation. No subagents are currently defined.\",\"name\":\"task\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"prompt\"],\"properties\":{\"description\":{\"type\":\"string\",\"description\":\"Short human-readable label for the delegated work\"},\"prompt\":{\"type\":\"string\",\"description\":\"Focused instructions for the child agent\"},\"agent\":{\"type\":\"string\",\"description\":\"Declared subagent to use for the child agent\"},\"cwd\":{\"type\":\"string\",\"description\":\"Working directory for the child agent. AGENTS.md and skills are discovered from here.\"}}},\"strict\":false}],\"top_logprobs\":0,\"top_p\":0.98,\"truncation\":\"disabled\",\"usage\":null,\"user\":null,\"metadata\":{}},\"sequence_number\":1}", - "event: response.output_item.added\ndata: {\"type\":\"response.output_item.added\",\"item\":{\"id\":\"fc_054a16a2f2ea3217016a705b68c39881a3b1e518599236e41a\",\"type\":\"function_call\",\"status\":\"in_progress\",\"arguments\":\"\",\"call_id\":\"call_NgMSUb6wIR0GAf9U8LXERtBQ\",\"name\":\"bash\"},\"output_index\":0,\"sequence_number\":2}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"{\\\"\",\"item_id\":\"fc_054a16a2f2ea3217016a705b68c39881a3b1e518599236e41a\",\"obfuscation\":\"Ecm4AJUnR33nzy\",\"output_index\":0,\"sequence_number\":3}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"command\",\"item_id\":\"fc_054a16a2f2ea3217016a705b68c39881a3b1e518599236e41a\",\"obfuscation\":\"qAhzrPXMU\",\"output_index\":0,\"sequence_number\":4}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"\\\":\\\"\",\"item_id\":\"fc_054a16a2f2ea3217016a705b68c39881a3b1e518599236e41a\",\"obfuscation\":\"HjoLYpLsOOApn\",\"output_index\":0,\"sequence_number\":5}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"find\",\"item_id\":\"fc_054a16a2f2ea3217016a705b68c39881a3b1e518599236e41a\",\"obfuscation\":\"COx5RXyau1iu\",\"output_index\":0,\"sequence_number\":6}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" /\",\"item_id\":\"fc_054a16a2f2ea3217016a705b68c39881a3b1e518599236e41a\",\"obfuscation\":\"7fcdZxijyLBFbj\",\"output_index\":0,\"sequence_number\":7}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"ver\",\"item_id\":\"fc_054a16a2f2ea3217016a705b68c39881a3b1e518599236e41a\",\"obfuscation\":\"EdhPBWD97UXXj\",\"output_index\":0,\"sequence_number\":8}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"cel\",\"item_id\":\"fc_054a16a2f2ea3217016a705b68c39881a3b1e518599236e41a\",\"obfuscation\":\"g4JtDEv0GyNkp\",\"output_index\":0,\"sequence_number\":9}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/s\",\"item_id\":\"fc_054a16a2f2ea3217016a705b68c39881a3b1e518599236e41a\",\"obfuscation\":\"Yu3ytbv7EfB9L7\",\"output_index\":0,\"sequence_number\":10}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"andbox\",\"item_id\":\"fc_054a16a2f2ea3217016a705b68c39881a3b1e518599236e41a\",\"obfuscation\":\"loDk0mviWC\",\"output_index\":0,\"sequence_number\":11}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/\",\"item_id\":\"fc_054a16a2f2ea3217016a705b68c39881a3b1e518599236e41a\",\"obfuscation\":\"4KyvKX7bONkWif0\",\"output_index\":0,\"sequence_number\":12}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"brain\",\"item_id\":\"fc_054a16a2f2ea3217016a705b68c39881a3b1e518599236e41a\",\"obfuscation\":\"yowGulOqnpo\",\"output_index\":0,\"sequence_number\":13}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"trust\",\"item_id\":\"fc_054a16a2f2ea3217016a705b68c39881a3b1e518599236e41a\",\"obfuscation\":\"83qIG2PGqLt\",\"output_index\":0,\"sequence_number\":14}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"data\",\"item_id\":\"fc_054a16a2f2ea3217016a705b68c39881a3b1e518599236e41a\",\"obfuscation\":\"ZFbal7RnszIt\",\"output_index\":0,\"sequence_number\":15}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/\",\"item_id\":\"fc_054a16a2f2ea3217016a705b68c39881a3b1e518599236e41a\",\"obfuscation\":\"GVlOdY86Xo86jAw\",\"output_index\":0,\"sequence_number\":16}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"brain\",\"item_id\":\"fc_054a16a2f2ea3217016a705b68c39881a3b1e518599236e41a\",\"obfuscation\":\"mjcerP27QOt\",\"output_index\":0,\"sequence_number\":17}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"trust\",\"item_id\":\"fc_054a16a2f2ea3217016a705b68c39881a3b1e518599236e41a\",\"obfuscation\":\"oQnyazxVeT6\",\"output_index\":0,\"sequence_number\":18}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"-sdk\",\"item_id\":\"fc_054a16a2f2ea3217016a705b68c39881a3b1e518599236e41a\",\"obfuscation\":\"2CRHfaUuS1Dd\",\"output_index\":0,\"sequence_number\":19}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"-j\",\"item_id\":\"fc_054a16a2f2ea3217016a705b68c39881a3b1e518599236e41a\",\"obfuscation\":\"3IfHRkUmRfpbWX\",\"output_index\":0,\"sequence_number\":20}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"avascript\",\"item_id\":\"fc_054a16a2f2ea3217016a705b68c39881a3b1e518599236e41a\",\"obfuscation\":\"uJPFidS\",\"output_index\":0,\"sequence_number\":21}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/e\",\"item_id\":\"fc_054a16a2f2ea3217016a705b68c39881a3b1e518599236e41a\",\"obfuscation\":\"PAbcespQ5gkoQu\",\"output_index\":0,\"sequence_number\":22}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"2\",\"item_id\":\"fc_054a16a2f2ea3217016a705b68c39881a3b1e518599236e41a\",\"obfuscation\":\"By2Tx9QLwPZRKd1\",\"output_index\":0,\"sequence_number\":23}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"e\",\"item_id\":\"fc_054a16a2f2ea3217016a705b68c39881a3b1e518599236e41a\",\"obfuscation\":\"jCZWJ7sTG0eCEsh\",\"output_index\":0,\"sequence_number\":24}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" -\",\"item_id\":\"fc_054a16a2f2ea3217016a705b68c39881a3b1e518599236e41a\",\"obfuscation\":\"gu7kgzv2IJ1Nyl\",\"output_index\":0,\"sequence_number\":25}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"max\",\"item_id\":\"fc_054a16a2f2ea3217016a705b68c39881a3b1e518599236e41a\",\"obfuscation\":\"ftZoJcWqYeSjP\",\"output_index\":0,\"sequence_number\":26}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"depth\",\"item_id\":\"fc_054a16a2f2ea3217016a705b68c39881a3b1e518599236e41a\",\"obfuscation\":\"pVAaXJ9DEpZ\",\"output_index\":0,\"sequence_number\":27}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" \",\"item_id\":\"fc_054a16a2f2ea3217016a705b68c39881a3b1e518599236e41a\",\"obfuscation\":\"sR3ARyMb8XihmRv\",\"output_index\":0,\"sequence_number\":28}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"6\",\"item_id\":\"fc_054a16a2f2ea3217016a705b68c39881a3b1e518599236e41a\",\"obfuscation\":\"O5oruVniSAltaiT\",\"output_index\":0,\"sequence_number\":29}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" -\",\"item_id\":\"fc_054a16a2f2ea3217016a705b68c39881a3b1e518599236e41a\",\"obfuscation\":\"vv0l4nT2EtqHMG\",\"output_index\":0,\"sequence_number\":30}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"type\",\"item_id\":\"fc_054a16a2f2ea3217016a705b68c39881a3b1e518599236e41a\",\"obfuscation\":\"eCQV3vrXZ37f\",\"output_index\":0,\"sequence_number\":31}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" f\",\"item_id\":\"fc_054a16a2f2ea3217016a705b68c39881a3b1e518599236e41a\",\"obfuscation\":\"LD2tChTMhpkm7O\",\"output_index\":0,\"sequence_number\":32}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" -\",\"item_id\":\"fc_054a16a2f2ea3217016a705b68c39881a3b1e518599236e41a\",\"obfuscation\":\"dbjx8JTI5wqqpk\",\"output_index\":0,\"sequence_number\":33}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"name\",\"item_id\":\"fc_054a16a2f2ea3217016a705b68c39881a3b1e518599236e41a\",\"obfuscation\":\"CMGMxsC84MqB\",\"output_index\":0,\"sequence_number\":34}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" '\",\"item_id\":\"fc_054a16a2f2ea3217016a705b68c39881a3b1e518599236e41a\",\"obfuscation\":\"IP4ilgz1kRzQlU\",\"output_index\":0,\"sequence_number\":35}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"AG\",\"item_id\":\"fc_054a16a2f2ea3217016a705b68c39881a3b1e518599236e41a\",\"obfuscation\":\"30S1wp0FSdH8j0\",\"output_index\":0,\"sequence_number\":36}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"ENTS\",\"item_id\":\"fc_054a16a2f2ea3217016a705b68c39881a3b1e518599236e41a\",\"obfuscation\":\"LbBUDVVtGwst\",\"output_index\":0,\"sequence_number\":37}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\".md\",\"item_id\":\"fc_054a16a2f2ea3217016a705b68c39881a3b1e518599236e41a\",\"obfuscation\":\"8piB3U9Zy7jyA\",\"output_index\":0,\"sequence_number\":38}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"'\",\"item_id\":\"fc_054a16a2f2ea3217016a705b68c39881a3b1e518599236e41a\",\"obfuscation\":\"9DRBJbRy6maC4gS\",\"output_index\":0,\"sequence_number\":39}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" -\",\"item_id\":\"fc_054a16a2f2ea3217016a705b68c39881a3b1e518599236e41a\",\"obfuscation\":\"XAt6MDPojUNwz3\",\"output_index\":0,\"sequence_number\":40}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"print\",\"item_id\":\"fc_054a16a2f2ea3217016a705b68c39881a3b1e518599236e41a\",\"obfuscation\":\"2sb7d6gE12q\",\"output_index\":0,\"sequence_number\":41}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"\\\",\\\"\",\"item_id\":\"fc_054a16a2f2ea3217016a705b68c39881a3b1e518599236e41a\",\"obfuscation\":\"78xkqC87iDA2l\",\"output_index\":0,\"sequence_number\":42}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"timeout\",\"item_id\":\"fc_054a16a2f2ea3217016a705b68c39881a3b1e518599236e41a\",\"obfuscation\":\"LZICbxiFz\",\"output_index\":0,\"sequence_number\":43}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"\\\":\",\"item_id\":\"fc_054a16a2f2ea3217016a705b68c39881a3b1e518599236e41a\",\"obfuscation\":\"L39ZgCy9F9L7ou\",\"output_index\":0,\"sequence_number\":44}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"100\",\"item_id\":\"fc_054a16a2f2ea3217016a705b68c39881a3b1e518599236e41a\",\"obfuscation\":\"cmBrFOna5ifiL\",\"output_index\":0,\"sequence_number\":45}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"000\",\"item_id\":\"fc_054a16a2f2ea3217016a705b68c39881a3b1e518599236e41a\",\"obfuscation\":\"z5qEzhvJIbUVN\",\"output_index\":0,\"sequence_number\":46}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"}\",\"item_id\":\"fc_054a16a2f2ea3217016a705b68c39881a3b1e518599236e41a\",\"obfuscation\":\"S29PMSSd6WVTIYN\",\"output_index\":0,\"sequence_number\":47}", - "event: response.function_call_arguments.done\ndata: {\"type\":\"response.function_call_arguments.done\",\"arguments\":\"{\\\"command\\\":\\\"find . -maxdepth 6 -type f -name 'AGENTS.md' -print\\\",\\\"timeout\\\":100000}\",\"item_id\":\"fc_054a16a2f2ea3217016a705b68c39881a3b1e518599236e41a\",\"output_index\":0,\"sequence_number\":48}", - "event: response.output_item.done\ndata: {\"type\":\"response.output_item.done\",\"item\":{\"id\":\"fc_054a16a2f2ea3217016a705b68c39881a3b1e518599236e41a\",\"type\":\"function_call\",\"status\":\"completed\",\"arguments\":\"{\\\"command\\\":\\\"find . -maxdepth 6 -type f -name 'AGENTS.md' -print\\\",\\\"timeout\\\":100000}\",\"call_id\":\"call_NgMSUb6wIR0GAf9U8LXERtBQ\",\"name\":\"bash\"},\"output_index\":0,\"sequence_number\":49}", - "event: response.completed\ndata: {\"type\":\"response.completed\",\"response\":{\"id\":\"resp_054a16a2f2ea3217016a705b6820c481a3b64f6527345dc3ca\",\"object\":\"response\",\"created_at\":1785748328,\"status\":\"completed\",\"background\":false,\"completed_at\":1785748329,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":null,\"max_tool_calls\":null,\"model\":\"gpt-5.4-nano-2026-03-17\",\"moderation\":null,\"output\":[{\"id\":\"fc_054a16a2f2ea3217016a705b68c39881a3b1e518599236e41a\",\"type\":\"function_call\",\"status\":\"completed\",\"arguments\":\"{\\\"command\\\":\\\"find . -maxdepth 6 -type f -name 'AGENTS.md' -print\\\",\\\"timeout\\\":100000}\",\"call_id\":\"call_NgMSUb6wIR0GAf9U8LXERtBQ\",\"name\":\"bash\"}],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":\"workflow:instrumentation:01KZ3E9TWTAEK4P4P3F63HKAKC::default::ta\",\"prompt_cache_retention\":\"24h\",\"reasoning\":{\"context\":\"current_turn\",\"effort\":\"none\",\"mode\":\"standard\",\"summary\":null},\"safety_identifier\":null,\"service_tier\":\"default\",\"store\":false,\"temperature\":1,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tool_usage\":{\"image_gen\":{\"input_tokens\":0,\"input_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"output_tokens\":0,\"output_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"total_tokens\":0},\"web_search\":{\"num_requests\":0}},\"tools\":[{\"type\":\"function\",\"description\":\"Read a file or list a directory. For files, output is truncated to 2000 lines or 50KB — use offset/limit for large files. For directories, returns the list of entries.\",\"name\":\"read\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to read\"},\"offset\":{\"type\":\"number\",\"description\":\"Line number to start from (1-indexed)\"},\"limit\":{\"type\":\"number\",\"description\":\"Maximum number of lines to read\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Write content to a file. Creates the file and parent directories if they do not exist.\",\"name\":\"write\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"content\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to write\"},\"content\":{\"type\":\"string\",\"description\":\"Content to write to the file\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Edit a file using exact text replacement. The oldText must match a unique region of the file. Use replaceAll to replace all occurrences.\",\"name\":\"edit\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"oldText\",\"newText\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to edit\"},\"oldText\":{\"type\":\"string\",\"description\":\"Exact text to find (must be unique)\"},\"newText\":{\"type\":\"string\",\"description\":\"Replacement text\"},\"replaceAll\":{\"type\":\"boolean\",\"description\":\"Replace all occurrences\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Execute a bash command. Returns stdout and stderr. Output is truncated to the last 2000 lines or 50KB.\",\"name\":\"bash\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"command\"],\"properties\":{\"command\":{\"type\":\"string\",\"description\":\"Bash command to execute\"},\"timeout\":{\"type\":\"number\",\"description\":\"Timeout in seconds\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Search file contents for a regex pattern. Returns matching lines with file paths and line numbers.\",\"name\":\"grep\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Search pattern (regex)\"},\"path\":{\"type\":\"string\",\"description\":\"Directory or file to search (default: .)\"},\"include\":{\"type\":\"string\",\"description\":\"Glob filter, e.g. \\\"*.ts\\\"\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Find files by filename pattern using shell find -name semantics. Returns matching file paths.\",\"name\":\"glob\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Filename pattern, e.g. \\\"*.ts\\\"\"},\"path\":{\"type\":\"string\",\"description\":\"Directory to search in (default: .)\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Delegate a focused task to a detached child agent with its own context. Use this for independent research, file exploration, or parallel work. The task returns only its final answer to this conversation. No subagents are currently defined.\",\"name\":\"task\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"prompt\"],\"properties\":{\"description\":{\"type\":\"string\",\"description\":\"Short human-readable label for the delegated work\"},\"prompt\":{\"type\":\"string\",\"description\":\"Focused instructions for the child agent\"},\"agent\":{\"type\":\"string\",\"description\":\"Declared subagent to use for the child agent\"},\"cwd\":{\"type\":\"string\",\"description\":\"Working directory for the child agent. AGENTS.md and skills are discovered from here.\"}}},\"strict\":false}],\"top_logprobs\":0,\"top_p\":0.98,\"truncation\":\"disabled\",\"usage\":{\"input_tokens\":1298,\"input_tokens_details\":{\"cache_write_tokens\":0,\"cached_tokens\":0},\"output_tokens\":58,\"output_tokens_details\":{\"reasoning_tokens\":0},\"total_tokens\":1356},\"user\":null,\"metadata\":{}},\"sequence_number\":50}" - ], - "kind": "sse" - }, - "headers": { - "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", - "alt-svc": "h3=\":443\"; ma=86400", - "cf-cache-status": "DYNAMIC", - "cf-ray": "a25432ea7f5d7bb6-IAD", - "connection": "keep-alive", - "content-type": "text/event-stream; charset=utf-8", - "date": "Mon, 03 Aug 2026 09:12:08 GMT", - "openai-organization": "braintrust-data", - "openai-processing-ms": "428", - "openai-project": "proj_vsCSXafhhByzWOThMrJcZiw9", - "openai-version": "2020-10-01", - "server": "cloudflare", - "set-cookie": "[REDACTED]", - "strict-transport-security": "max-age=31536000; includeSubDomains; preload", - "transfer-encoding": "chunked", - "x-content-type-options": "nosniff", - "x-ratelimit-limit-requests": "30000", - "x-ratelimit-limit-tokens": "180000000", - "x-ratelimit-remaining-requests": "29999", - "x-ratelimit-remaining-tokens": "179998245", - "x-ratelimit-reset-requests": "2ms", - "x-ratelimit-reset-tokens": "0s", - "x-request-id": "req_e00fc12bdba24599905386f92e325433" - }, - "status": 200, - "statusText": "OK" - } - }, - { - "callIndex": 9, - "id": "6945bf18d345ad4a", - "matchKey": "POST api.openai.com/v1/responses", - "recordedAt": "2026-08-03T09:12:10.503Z", - "request": { - "body": { - "kind": "json", - "value": { - "input": [ - { - "content": "You are running in headless mode with no human operator. Work autonomously — never ask questions, never wait for user input. Make your best judgment and proceed independently.\n\nYou are a deterministic Flue instrumentation test agent. Follow user instructions exactly. When asked for a marker, output only that marker and no extra text. When running a local skill file, read it yourself and do not delegate it to a task.\n\n## Available Skills\n\nThe following skills provide specialized instructions for specific tasks. When a task matches a skill description, activate that skill before proceeding so its full instructions are loaded. Skill instructions and supporting resources stay lazy until activation or explicit file reads.\n\n- **e2e-flue-skill** — Deterministic skill used by the Braintrust Flue instrumentation e2e test.\n\nDate: Mon, Aug 3, 2026\nWorking directory: .\n\nDirectory structure:\n.agents\n.flue\n.flue-build\n__cassettes__\n__snapshots__\nassertions.ts\ncassette-filter.mjs\nconstants.mjs\ndist\nnode_modules\npackage.json\npnpm-lock.yaml\nscenario.cli.mjs\nscenario.impl.mjs\nscenario.mjs\nscenario.test.ts\nscenario.ts\nscenario.v2.ts\nv2-assertions.ts\nversions", - "role": "developer" - }, - { - "content": [ - { - "text": "Run the skill named \"e2e-flue-skill\" with arguments {\"marker\":\"SKILL_DONE\"}. Output only the marker and no extra text when finished.", - "type": "input_text" - } - ], - "role": "user" - }, - { - "arguments": "{\"description\":\"Run e2e-flue-skill with marker SKILL_DONE\",\"prompt\":\"Run the skill named \\\"e2e-flue-skill\\\" with arguments {\\\"marker\\\":\\\"SKILL_DONE\\\"}. Ensure completion and report the final output marker only.\",\"agent\":\"e2e-flue-skill\",\"cwd\":\".\"}", - "call_id": "call_w6pyYZgnhlSOAXul75DAG1Cj", - "id": "fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6", - "name": "task", - "type": "function_call" - }, - { - "call_id": "call_w6pyYZgnhlSOAXul75DAG1Cj", - "output": "[flue] Subagent \"e2e-flue-skill\" is not declared. Available: (none).", - "type": "function_call_output" - }, - { - "arguments": "{\"command\":\"find . -maxdepth 4 -name AGENTS.md -print\",\"timeout\":100000}", - "call_id": "call_6zEBnJRmXFIkGFwNjWLZP4uC", - "id": "fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee", - "name": "bash", - "type": "function_call" - }, - { - "call_id": "call_6zEBnJRmXFIkGFwNjWLZP4uC", - "output": "total 4\ndrwxr-xr-x 3 vercel-sandbox vercel-sandbox 20 Aug 3 09:10 .\ndrwxr-xr-x 10 vercel-sandbox vercel-sandbox 4096 Aug 3 09:11 ..\ndrwxr-xr-x 3 vercel-sandbox vercel-sandbox 28 Aug 3 09:10 skills", - "type": "function_call_output" - }, - { - "arguments": "{\"command\":\"find . -maxdepth 4 -type f -name AGENTS.md -print -exec sed -n '1,200p' {} \\\\;\",\"timeout\":100000}", - "call_id": "call_jQUCie0ZgaXUrZCDcezp0msl", - "id": "fc_054a16a2f2ea3217016a705b67575881a3aacde02b9053a321", - "name": "bash", - "type": "function_call" - }, - { - "call_id": "call_jQUCie0ZgaXUrZCDcezp0msl", - "output": "(no output)", - "type": "function_call_output" - }, - { - "arguments": "{\"command\":\"find . -maxdepth 6 -type f -name 'AGENTS.md' -print\",\"timeout\":100000}", - "call_id": "call_NgMSUb6wIR0GAf9U8LXERtBQ", - "id": "fc_054a16a2f2ea3217016a705b68c39881a3b1e518599236e41a", - "name": "bash", - "type": "function_call" - }, - { - "call_id": "call_NgMSUb6wIR0GAf9U8LXERtBQ", - "output": "(no output)", - "type": "function_call_output" - } - ], - "model": "gpt-5.4-nano", - "prompt_cache_key": "[REDACTED]", - "reasoning": { - "effort": "none" - }, - "store": false, - "stream": true, - "tools": [ - { - "description": "Read a file or list a directory. For files, output is truncated to 2000 lines or 50KB — use offset/limit for large files. For directories, returns the list of entries.", - "name": "read", - "parameters": { - "properties": { - "limit": { - "description": "Maximum number of lines to read", - "type": "number" - }, - "offset": { - "description": "Line number to start from (1-indexed)", - "type": "number" - }, - "path": { - "description": "Path to the file to read", - "type": "string" - } - }, - "required": ["path"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Write content to a file. Creates the file and parent directories if they do not exist.", - "name": "write", - "parameters": { - "properties": { - "content": { - "description": "Content to write to the file", - "type": "string" - }, - "path": { - "description": "Path to the file to write", - "type": "string" - } - }, - "required": ["path", "content"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Edit a file using exact text replacement. The oldText must match a unique region of the file. Use replaceAll to replace all occurrences.", - "name": "edit", - "parameters": { - "properties": { - "newText": { - "description": "Replacement text", - "type": "string" - }, - "oldText": { - "description": "Exact text to find (must be unique)", - "type": "string" - }, - "path": { - "description": "Path to the file to edit", - "type": "string" - }, - "replaceAll": { - "description": "Replace all occurrences", - "type": "boolean" - } - }, - "required": ["path", "oldText", "newText"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Execute a bash command. Returns stdout and stderr. Output is truncated to the last 2000 lines or 50KB.", - "name": "bash", - "parameters": { - "properties": { - "command": { - "description": "Bash command to execute", - "type": "string" - }, - "timeout": { - "description": "Timeout in seconds", - "type": "number" - } - }, - "required": ["command"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Search file contents for a regex pattern. Returns matching lines with file paths and line numbers.", - "name": "grep", - "parameters": { - "properties": { - "include": { - "description": "Glob filter, e.g. \"*.ts\"", - "type": "string" - }, - "path": { - "description": "Directory or file to search (default: .)", - "type": "string" - }, - "pattern": { - "description": "Search pattern (regex)", - "type": "string" - } - }, - "required": ["pattern"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Find files by filename pattern using shell find -name semantics. Returns matching file paths.", - "name": "glob", - "parameters": { - "properties": { - "path": { - "description": "Directory to search in (default: .)", - "type": "string" - }, - "pattern": { - "description": "Filename pattern, e.g. \"*.ts\"", - "type": "string" - } - }, - "required": ["pattern"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Delegate a focused task to a detached child agent with its own context. Use this for independent research, file exploration, or parallel work. The task returns only its final answer to this conversation. No subagents are currently defined.", - "name": "task", - "parameters": { - "properties": { - "agent": { - "description": "Declared subagent to use for the child agent", - "type": "string" - }, - "cwd": { - "description": "Working directory for the child agent. AGENTS.md and skills are discovered from here.", - "type": "string" - }, - "description": { - "description": "Short human-readable label for the delegated work", - "type": "string" - }, - "prompt": { - "description": "Focused instructions for the child agent", - "type": "string" - } - }, - "required": ["prompt"], - "type": "object" - }, - "strict": false, - "type": "function" - } - ] - } - }, - "headers": {}, - "method": "POST", - "url": "https://api.openai.com/v1/responses" - }, - "response": { - "body": { - "chunks": [ - "event: response.created\ndata: {\"type\":\"response.created\",\"response\":{\"id\":\"resp_054a16a2f2ea3217016a705b69681081a381f7cdcf2e5f8c38\",\"object\":\"response\",\"created_at\":1785748329,\"status\":\"in_progress\",\"background\":false,\"completed_at\":null,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":null,\"max_tool_calls\":null,\"model\":\"gpt-5.4-nano-2026-03-17\",\"moderation\":null,\"output\":[],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":\"workflow:instrumentation:01KZ3E9TWTAEK4P4P3F63HKAKC::default::ta\",\"prompt_cache_retention\":\"24h\",\"reasoning\":{\"context\":\"current_turn\",\"effort\":\"none\",\"mode\":\"standard\",\"summary\":null},\"safety_identifier\":null,\"service_tier\":\"auto\",\"store\":false,\"temperature\":1,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tool_usage\":{\"image_gen\":{\"input_tokens\":0,\"input_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"output_tokens\":0,\"output_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"total_tokens\":0},\"web_search\":{\"num_requests\":0}},\"tools\":[{\"type\":\"function\",\"description\":\"Read a file or list a directory. For files, output is truncated to 2000 lines or 50KB — use offset/limit for large files. For directories, returns the list of entries.\",\"name\":\"read\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to read\"},\"offset\":{\"type\":\"number\",\"description\":\"Line number to start from (1-indexed)\"},\"limit\":{\"type\":\"number\",\"description\":\"Maximum number of lines to read\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Write content to a file. Creates the file and parent directories if they do not exist.\",\"name\":\"write\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"content\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to write\"},\"content\":{\"type\":\"string\",\"description\":\"Content to write to the file\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Edit a file using exact text replacement. The oldText must match a unique region of the file. Use replaceAll to replace all occurrences.\",\"name\":\"edit\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"oldText\",\"newText\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to edit\"},\"oldText\":{\"type\":\"string\",\"description\":\"Exact text to find (must be unique)\"},\"newText\":{\"type\":\"string\",\"description\":\"Replacement text\"},\"replaceAll\":{\"type\":\"boolean\",\"description\":\"Replace all occurrences\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Execute a bash command. Returns stdout and stderr. Output is truncated to the last 2000 lines or 50KB.\",\"name\":\"bash\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"command\"],\"properties\":{\"command\":{\"type\":\"string\",\"description\":\"Bash command to execute\"},\"timeout\":{\"type\":\"number\",\"description\":\"Timeout in seconds\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Search file contents for a regex pattern. Returns matching lines with file paths and line numbers.\",\"name\":\"grep\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Search pattern (regex)\"},\"path\":{\"type\":\"string\",\"description\":\"Directory or file to search (default: .)\"},\"include\":{\"type\":\"string\",\"description\":\"Glob filter, e.g. \\\"*.ts\\\"\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Find files by filename pattern using shell find -name semantics. Returns matching file paths.\",\"name\":\"glob\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Filename pattern, e.g. \\\"*.ts\\\"\"},\"path\":{\"type\":\"string\",\"description\":\"Directory to search in (default: .)\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Delegate a focused task to a detached child agent with its own context. Use this for independent research, file exploration, or parallel work. The task returns only its final answer to this conversation. No subagents are currently defined.\",\"name\":\"task\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"prompt\"],\"properties\":{\"description\":{\"type\":\"string\",\"description\":\"Short human-readable label for the delegated work\"},\"prompt\":{\"type\":\"string\",\"description\":\"Focused instructions for the child agent\"},\"agent\":{\"type\":\"string\",\"description\":\"Declared subagent to use for the child agent\"},\"cwd\":{\"type\":\"string\",\"description\":\"Working directory for the child agent. AGENTS.md and skills are discovered from here.\"}}},\"strict\":false}],\"top_logprobs\":0,\"top_p\":0.98,\"truncation\":\"disabled\",\"usage\":null,\"user\":null,\"metadata\":{}},\"sequence_number\":0}", - "event: response.in_progress\ndata: {\"type\":\"response.in_progress\",\"response\":{\"id\":\"resp_054a16a2f2ea3217016a705b69681081a381f7cdcf2e5f8c38\",\"object\":\"response\",\"created_at\":1785748329,\"status\":\"in_progress\",\"background\":false,\"completed_at\":null,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":null,\"max_tool_calls\":null,\"model\":\"gpt-5.4-nano-2026-03-17\",\"moderation\":null,\"output\":[],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":\"workflow:instrumentation:01KZ3E9TWTAEK4P4P3F63HKAKC::default::ta\",\"prompt_cache_retention\":\"24h\",\"reasoning\":{\"context\":\"current_turn\",\"effort\":\"none\",\"mode\":\"standard\",\"summary\":null},\"safety_identifier\":null,\"service_tier\":\"auto\",\"store\":false,\"temperature\":1,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tool_usage\":{\"image_gen\":{\"input_tokens\":0,\"input_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"output_tokens\":0,\"output_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"total_tokens\":0},\"web_search\":{\"num_requests\":0}},\"tools\":[{\"type\":\"function\",\"description\":\"Read a file or list a directory. For files, output is truncated to 2000 lines or 50KB — use offset/limit for large files. For directories, returns the list of entries.\",\"name\":\"read\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to read\"},\"offset\":{\"type\":\"number\",\"description\":\"Line number to start from (1-indexed)\"},\"limit\":{\"type\":\"number\",\"description\":\"Maximum number of lines to read\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Write content to a file. Creates the file and parent directories if they do not exist.\",\"name\":\"write\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"content\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to write\"},\"content\":{\"type\":\"string\",\"description\":\"Content to write to the file\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Edit a file using exact text replacement. The oldText must match a unique region of the file. Use replaceAll to replace all occurrences.\",\"name\":\"edit\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"oldText\",\"newText\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to edit\"},\"oldText\":{\"type\":\"string\",\"description\":\"Exact text to find (must be unique)\"},\"newText\":{\"type\":\"string\",\"description\":\"Replacement text\"},\"replaceAll\":{\"type\":\"boolean\",\"description\":\"Replace all occurrences\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Execute a bash command. Returns stdout and stderr. Output is truncated to the last 2000 lines or 50KB.\",\"name\":\"bash\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"command\"],\"properties\":{\"command\":{\"type\":\"string\",\"description\":\"Bash command to execute\"},\"timeout\":{\"type\":\"number\",\"description\":\"Timeout in seconds\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Search file contents for a regex pattern. Returns matching lines with file paths and line numbers.\",\"name\":\"grep\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Search pattern (regex)\"},\"path\":{\"type\":\"string\",\"description\":\"Directory or file to search (default: .)\"},\"include\":{\"type\":\"string\",\"description\":\"Glob filter, e.g. \\\"*.ts\\\"\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Find files by filename pattern using shell find -name semantics. Returns matching file paths.\",\"name\":\"glob\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Filename pattern, e.g. \\\"*.ts\\\"\"},\"path\":{\"type\":\"string\",\"description\":\"Directory to search in (default: .)\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Delegate a focused task to a detached child agent with its own context. Use this for independent research, file exploration, or parallel work. The task returns only its final answer to this conversation. No subagents are currently defined.\",\"name\":\"task\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"prompt\"],\"properties\":{\"description\":{\"type\":\"string\",\"description\":\"Short human-readable label for the delegated work\"},\"prompt\":{\"type\":\"string\",\"description\":\"Focused instructions for the child agent\"},\"agent\":{\"type\":\"string\",\"description\":\"Declared subagent to use for the child agent\"},\"cwd\":{\"type\":\"string\",\"description\":\"Working directory for the child agent. AGENTS.md and skills are discovered from here.\"}}},\"strict\":false}],\"top_logprobs\":0,\"top_p\":0.98,\"truncation\":\"disabled\",\"usage\":null,\"user\":null,\"metadata\":{}},\"sequence_number\":1}", - "event: response.output_item.added\ndata: {\"type\":\"response.output_item.added\",\"item\":{\"id\":\"fc_054a16a2f2ea3217016a705b69edd081a38c31583ec56eca5f\",\"type\":\"function_call\",\"status\":\"in_progress\",\"arguments\":\"\",\"call_id\":\"call_BXKiQ5KG8c42XPPp94zgcY3H\",\"name\":\"bash\"},\"output_index\":0,\"sequence_number\":2}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"{\\\"\",\"item_id\":\"fc_054a16a2f2ea3217016a705b69edd081a38c31583ec56eca5f\",\"obfuscation\":\"hG7XwE8kTftCFD\",\"output_index\":0,\"sequence_number\":3}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"command\",\"item_id\":\"fc_054a16a2f2ea3217016a705b69edd081a38c31583ec56eca5f\",\"obfuscation\":\"SAESyTCDU\",\"output_index\":0,\"sequence_number\":4}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"\\\":\\\"\",\"item_id\":\"fc_054a16a2f2ea3217016a705b69edd081a38c31583ec56eca5f\",\"obfuscation\":\"FdSHAU6VPfEiP\",\"output_index\":0,\"sequence_number\":5}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"find\",\"item_id\":\"fc_054a16a2f2ea3217016a705b69edd081a38c31583ec56eca5f\",\"obfuscation\":\"JTitwOdHc00P\",\"output_index\":0,\"sequence_number\":6}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" /\",\"item_id\":\"fc_054a16a2f2ea3217016a705b69edd081a38c31583ec56eca5f\",\"obfuscation\":\"Z5ba86g8nOxzKr\",\"output_index\":0,\"sequence_number\":7}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"ver\",\"item_id\":\"fc_054a16a2f2ea3217016a705b69edd081a38c31583ec56eca5f\",\"obfuscation\":\"BUYeIHkrw1ca3\",\"output_index\":0,\"sequence_number\":8}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"cel\",\"item_id\":\"fc_054a16a2f2ea3217016a705b69edd081a38c31583ec56eca5f\",\"obfuscation\":\"Jnlz6e6YIi8An\",\"output_index\":0,\"sequence_number\":9}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/s\",\"item_id\":\"fc_054a16a2f2ea3217016a705b69edd081a38c31583ec56eca5f\",\"obfuscation\":\"EiAxOlmPQ6R9Lk\",\"output_index\":0,\"sequence_number\":10}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"andbox\",\"item_id\":\"fc_054a16a2f2ea3217016a705b69edd081a38c31583ec56eca5f\",\"obfuscation\":\"HLwMNKLqXd\",\"output_index\":0,\"sequence_number\":11}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/\",\"item_id\":\"fc_054a16a2f2ea3217016a705b69edd081a38c31583ec56eca5f\",\"obfuscation\":\"CyMa169D7RtZmdr\",\"output_index\":0,\"sequence_number\":12}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"brain\",\"item_id\":\"fc_054a16a2f2ea3217016a705b69edd081a38c31583ec56eca5f\",\"obfuscation\":\"eBBwyrijOzo\",\"output_index\":0,\"sequence_number\":13}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"trust\",\"item_id\":\"fc_054a16a2f2ea3217016a705b69edd081a38c31583ec56eca5f\",\"obfuscation\":\"9oZJd40vJbv\",\"output_index\":0,\"sequence_number\":14}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"data\",\"item_id\":\"fc_054a16a2f2ea3217016a705b69edd081a38c31583ec56eca5f\",\"obfuscation\":\"au0nu8FGh24U\",\"output_index\":0,\"sequence_number\":15}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/\",\"item_id\":\"fc_054a16a2f2ea3217016a705b69edd081a38c31583ec56eca5f\",\"obfuscation\":\"mH2gvFHGB1gaA18\",\"output_index\":0,\"sequence_number\":16}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"brain\",\"item_id\":\"fc_054a16a2f2ea3217016a705b69edd081a38c31583ec56eca5f\",\"obfuscation\":\"W1iZL4cRSMq\",\"output_index\":0,\"sequence_number\":17}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"trust\",\"item_id\":\"fc_054a16a2f2ea3217016a705b69edd081a38c31583ec56eca5f\",\"obfuscation\":\"QUUnH5ISGOi\",\"output_index\":0,\"sequence_number\":18}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"-sdk\",\"item_id\":\"fc_054a16a2f2ea3217016a705b69edd081a38c31583ec56eca5f\",\"obfuscation\":\"eeLPcF9XcrHV\",\"output_index\":0,\"sequence_number\":19}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"-j\",\"item_id\":\"fc_054a16a2f2ea3217016a705b69edd081a38c31583ec56eca5f\",\"obfuscation\":\"lJ7n3mjPmhDNxo\",\"output_index\":0,\"sequence_number\":20}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"avascript\",\"item_id\":\"fc_054a16a2f2ea3217016a705b69edd081a38c31583ec56eca5f\",\"obfuscation\":\"Cq8la1K\",\"output_index\":0,\"sequence_number\":21}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/e\",\"item_id\":\"fc_054a16a2f2ea3217016a705b69edd081a38c31583ec56eca5f\",\"obfuscation\":\"QCMTFUK5pMEVRm\",\"output_index\":0,\"sequence_number\":22}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"2\",\"item_id\":\"fc_054a16a2f2ea3217016a705b69edd081a38c31583ec56eca5f\",\"obfuscation\":\"BgRGVixu3UbxQDA\",\"output_index\":0,\"sequence_number\":23}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"e\",\"item_id\":\"fc_054a16a2f2ea3217016a705b69edd081a38c31583ec56eca5f\",\"obfuscation\":\"aqbUU7TCMgXGNus\",\"output_index\":0,\"sequence_number\":24}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" -\",\"item_id\":\"fc_054a16a2f2ea3217016a705b69edd081a38c31583ec56eca5f\",\"obfuscation\":\"0aahbmWnQc34bI\",\"output_index\":0,\"sequence_number\":25}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"max\",\"item_id\":\"fc_054a16a2f2ea3217016a705b69edd081a38c31583ec56eca5f\",\"obfuscation\":\"hLd8UqgGfFZ9D\",\"output_index\":0,\"sequence_number\":26}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"depth\",\"item_id\":\"fc_054a16a2f2ea3217016a705b69edd081a38c31583ec56eca5f\",\"obfuscation\":\"GmhwY6EWAHh\",\"output_index\":0,\"sequence_number\":27}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" \",\"item_id\":\"fc_054a16a2f2ea3217016a705b69edd081a38c31583ec56eca5f\",\"obfuscation\":\"NRuEQgh0p3eRHWo\",\"output_index\":0,\"sequence_number\":28}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"5\",\"item_id\":\"fc_054a16a2f2ea3217016a705b69edd081a38c31583ec56eca5f\",\"obfuscation\":\"VSUdF5XfszF7J7C\",\"output_index\":0,\"sequence_number\":29}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" -\",\"item_id\":\"fc_054a16a2f2ea3217016a705b69edd081a38c31583ec56eca5f\",\"obfuscation\":\"7bgbWInYXsOzEZ\",\"output_index\":0,\"sequence_number\":30}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"type\",\"item_id\":\"fc_054a16a2f2ea3217016a705b69edd081a38c31583ec56eca5f\",\"obfuscation\":\"ZdXd8MYxzBVa\",\"output_index\":0,\"sequence_number\":31}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" f\",\"item_id\":\"fc_054a16a2f2ea3217016a705b69edd081a38c31583ec56eca5f\",\"obfuscation\":\"lKHlgmcHv4yQ7n\",\"output_index\":0,\"sequence_number\":32}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" -\",\"item_id\":\"fc_054a16a2f2ea3217016a705b69edd081a38c31583ec56eca5f\",\"obfuscation\":\"O3v0EDtKmgDYYi\",\"output_index\":0,\"sequence_number\":33}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"name\",\"item_id\":\"fc_054a16a2f2ea3217016a705b69edd081a38c31583ec56eca5f\",\"obfuscation\":\"yOsJv2sjpuJT\",\"output_index\":0,\"sequence_number\":34}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" '*\",\"item_id\":\"fc_054a16a2f2ea3217016a705b69edd081a38c31583ec56eca5f\",\"obfuscation\":\"bT4zdeKIYUfHc\",\"output_index\":0,\"sequence_number\":35}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"fl\",\"item_id\":\"fc_054a16a2f2ea3217016a705b69edd081a38c31583ec56eca5f\",\"obfuscation\":\"jujglZGbsBwPa7\",\"output_index\":0,\"sequence_number\":36}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"ue\",\"item_id\":\"fc_054a16a2f2ea3217016a705b69edd081a38c31583ec56eca5f\",\"obfuscation\":\"LKKBJiCjnXh4xf\",\"output_index\":0,\"sequence_number\":37}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"*\",\"item_id\":\"fc_054a16a2f2ea3217016a705b69edd081a38c31583ec56eca5f\",\"obfuscation\":\"shy3qPjXsnqPzfn\",\"output_index\":0,\"sequence_number\":38}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"skill\",\"item_id\":\"fc_054a16a2f2ea3217016a705b69edd081a38c31583ec56eca5f\",\"obfuscation\":\"QCXNC1UuV7e\",\"output_index\":0,\"sequence_number\":39}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"*\",\"item_id\":\"fc_054a16a2f2ea3217016a705b69edd081a38c31583ec56eca5f\",\"obfuscation\":\"XG8mVOll3Ohqe18\",\"output_index\":0,\"sequence_number\":40}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"'\",\"item_id\":\"fc_054a16a2f2ea3217016a705b69edd081a38c31583ec56eca5f\",\"obfuscation\":\"twOToqPAOsKMMVy\",\"output_index\":0,\"sequence_number\":41}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" -\",\"item_id\":\"fc_054a16a2f2ea3217016a705b69edd081a38c31583ec56eca5f\",\"obfuscation\":\"DZtAspDqcPPg83\",\"output_index\":0,\"sequence_number\":42}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"o\",\"item_id\":\"fc_054a16a2f2ea3217016a705b69edd081a38c31583ec56eca5f\",\"obfuscation\":\"1vqyG924TjjeMm1\",\"output_index\":0,\"sequence_number\":43}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" -\",\"item_id\":\"fc_054a16a2f2ea3217016a705b69edd081a38c31583ec56eca5f\",\"obfuscation\":\"vbMfw1I6yQT071\",\"output_index\":0,\"sequence_number\":44}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"name\",\"item_id\":\"fc_054a16a2f2ea3217016a705b69edd081a38c31583ec56eca5f\",\"obfuscation\":\"yEkTfUjLa7T3\",\"output_index\":0,\"sequence_number\":45}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" '*\",\"item_id\":\"fc_054a16a2f2ea3217016a705b69edd081a38c31583ec56eca5f\",\"obfuscation\":\"QbcNOxt5ouCjy\",\"output_index\":0,\"sequence_number\":46}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"skill\",\"item_id\":\"fc_054a16a2f2ea3217016a705b69edd081a38c31583ec56eca5f\",\"obfuscation\":\"fkrT1lk0jv9\",\"output_index\":0,\"sequence_number\":47}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"*\",\"item_id\":\"fc_054a16a2f2ea3217016a705b69edd081a38c31583ec56eca5f\",\"obfuscation\":\"9RVtGG8sxWEOjJr\",\"output_index\":0,\"sequence_number\":48}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"'\",\"item_id\":\"fc_054a16a2f2ea3217016a705b69edd081a38c31583ec56eca5f\",\"obfuscation\":\"Za4XJQ2Z2IsOIeb\",\"output_index\":0,\"sequence_number\":49}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" |\",\"item_id\":\"fc_054a16a2f2ea3217016a705b69edd081a38c31583ec56eca5f\",\"obfuscation\":\"px85R0MdttFruZ\",\"output_index\":0,\"sequence_number\":50}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" head\",\"item_id\":\"fc_054a16a2f2ea3217016a705b69edd081a38c31583ec56eca5f\",\"obfuscation\":\"Y88f7Ld1Ora\",\"output_index\":0,\"sequence_number\":51}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"\\\",\\\"\",\"item_id\":\"fc_054a16a2f2ea3217016a705b69edd081a38c31583ec56eca5f\",\"obfuscation\":\"8moCEbjvgTG1n\",\"output_index\":0,\"sequence_number\":52}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"timeout\",\"item_id\":\"fc_054a16a2f2ea3217016a705b69edd081a38c31583ec56eca5f\",\"obfuscation\":\"Kmyh4pQvd\",\"output_index\":0,\"sequence_number\":53}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"\\\":\",\"item_id\":\"fc_054a16a2f2ea3217016a705b69edd081a38c31583ec56eca5f\",\"obfuscation\":\"GrJ0KmP1JEa4oC\",\"output_index\":0,\"sequence_number\":54}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"100\",\"item_id\":\"fc_054a16a2f2ea3217016a705b69edd081a38c31583ec56eca5f\",\"obfuscation\":\"tQ7b2AdVP1fwn\",\"output_index\":0,\"sequence_number\":55}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"000\",\"item_id\":\"fc_054a16a2f2ea3217016a705b69edd081a38c31583ec56eca5f\",\"obfuscation\":\"OGHfUzdqw3zns\",\"output_index\":0,\"sequence_number\":56}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"}\",\"item_id\":\"fc_054a16a2f2ea3217016a705b69edd081a38c31583ec56eca5f\",\"obfuscation\":\"4USiuoYvOOTVjiY\",\"output_index\":0,\"sequence_number\":57}", - "event: response.function_call_arguments.done\ndata: {\"type\":\"response.function_call_arguments.done\",\"arguments\":\"{\\\"command\\\":\\\"find . -maxdepth 5 -type f -name '*flue*skill*' -o -name '*skill*' | head\\\",\\\"timeout\\\":100000}\",\"item_id\":\"fc_054a16a2f2ea3217016a705b69edd081a38c31583ec56eca5f\",\"output_index\":0,\"sequence_number\":58}", - "event: response.output_item.done\ndata: {\"type\":\"response.output_item.done\",\"item\":{\"id\":\"fc_054a16a2f2ea3217016a705b69edd081a38c31583ec56eca5f\",\"type\":\"function_call\",\"status\":\"completed\",\"arguments\":\"{\\\"command\\\":\\\"find . -maxdepth 5 -type f -name '*flue*skill*' -o -name '*skill*' | head\\\",\\\"timeout\\\":100000}\",\"call_id\":\"call_BXKiQ5KG8c42XPPp94zgcY3H\",\"name\":\"bash\"},\"output_index\":0,\"sequence_number\":59}", - "event: response.completed\ndata: {\"type\":\"response.completed\",\"response\":{\"id\":\"resp_054a16a2f2ea3217016a705b69681081a381f7cdcf2e5f8c38\",\"object\":\"response\",\"created_at\":1785748329,\"status\":\"completed\",\"background\":false,\"completed_at\":1785748330,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":null,\"max_tool_calls\":null,\"model\":\"gpt-5.4-nano-2026-03-17\",\"moderation\":null,\"output\":[{\"id\":\"fc_054a16a2f2ea3217016a705b69edd081a38c31583ec56eca5f\",\"type\":\"function_call\",\"status\":\"completed\",\"arguments\":\"{\\\"command\\\":\\\"find . -maxdepth 5 -type f -name '*flue*skill*' -o -name '*skill*' | head\\\",\\\"timeout\\\":100000}\",\"call_id\":\"call_BXKiQ5KG8c42XPPp94zgcY3H\",\"name\":\"bash\"}],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":\"workflow:instrumentation:01KZ3E9TWTAEK4P4P3F63HKAKC::default::ta\",\"prompt_cache_retention\":\"24h\",\"reasoning\":{\"context\":\"current_turn\",\"effort\":\"none\",\"mode\":\"standard\",\"summary\":null},\"safety_identifier\":null,\"service_tier\":\"default\",\"store\":false,\"temperature\":1,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tool_usage\":{\"image_gen\":{\"input_tokens\":0,\"input_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"output_tokens\":0,\"output_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"total_tokens\":0},\"web_search\":{\"num_requests\":0}},\"tools\":[{\"type\":\"function\",\"description\":\"Read a file or list a directory. For files, output is truncated to 2000 lines or 50KB — use offset/limit for large files. For directories, returns the list of entries.\",\"name\":\"read\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to read\"},\"offset\":{\"type\":\"number\",\"description\":\"Line number to start from (1-indexed)\"},\"limit\":{\"type\":\"number\",\"description\":\"Maximum number of lines to read\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Write content to a file. Creates the file and parent directories if they do not exist.\",\"name\":\"write\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"content\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to write\"},\"content\":{\"type\":\"string\",\"description\":\"Content to write to the file\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Edit a file using exact text replacement. The oldText must match a unique region of the file. Use replaceAll to replace all occurrences.\",\"name\":\"edit\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"oldText\",\"newText\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to edit\"},\"oldText\":{\"type\":\"string\",\"description\":\"Exact text to find (must be unique)\"},\"newText\":{\"type\":\"string\",\"description\":\"Replacement text\"},\"replaceAll\":{\"type\":\"boolean\",\"description\":\"Replace all occurrences\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Execute a bash command. Returns stdout and stderr. Output is truncated to the last 2000 lines or 50KB.\",\"name\":\"bash\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"command\"],\"properties\":{\"command\":{\"type\":\"string\",\"description\":\"Bash command to execute\"},\"timeout\":{\"type\":\"number\",\"description\":\"Timeout in seconds\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Search file contents for a regex pattern. Returns matching lines with file paths and line numbers.\",\"name\":\"grep\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Search pattern (regex)\"},\"path\":{\"type\":\"string\",\"description\":\"Directory or file to search (default: .)\"},\"include\":{\"type\":\"string\",\"description\":\"Glob filter, e.g. \\\"*.ts\\\"\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Find files by filename pattern using shell find -name semantics. Returns matching file paths.\",\"name\":\"glob\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Filename pattern, e.g. \\\"*.ts\\\"\"},\"path\":{\"type\":\"string\",\"description\":\"Directory to search in (default: .)\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Delegate a focused task to a detached child agent with its own context. Use this for independent research, file exploration, or parallel work. The task returns only its final answer to this conversation. No subagents are currently defined.\",\"name\":\"task\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"prompt\"],\"properties\":{\"description\":{\"type\":\"string\",\"description\":\"Short human-readable label for the delegated work\"},\"prompt\":{\"type\":\"string\",\"description\":\"Focused instructions for the child agent\"},\"agent\":{\"type\":\"string\",\"description\":\"Declared subagent to use for the child agent\"},\"cwd\":{\"type\":\"string\",\"description\":\"Working directory for the child agent. AGENTS.md and skills are discovered from here.\"}}},\"strict\":false}],\"top_logprobs\":0,\"top_p\":0.98,\"truncation\":\"disabled\",\"usage\":{\"input_tokens\":1370,\"input_tokens_details\":{\"cache_write_tokens\":0,\"cached_tokens\":0},\"output_tokens\":68,\"output_tokens_details\":{\"reasoning_tokens\":0},\"total_tokens\":1438},\"user\":null,\"metadata\":{}},\"sequence_number\":60}" - ], - "kind": "sse" - }, - "headers": { - "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", - "alt-svc": "h3=\":443\"; ma=86400", - "cf-cache-status": "DYNAMIC", - "cf-ray": "a25432f26f707bb6-IAD", - "connection": "keep-alive", - "content-type": "text/event-stream; charset=utf-8", - "date": "Mon, 03 Aug 2026 09:12:09 GMT", - "openai-organization": "braintrust-data", - "openai-processing-ms": "283", - "openai-project": "proj_vsCSXafhhByzWOThMrJcZiw9", - "openai-version": "2020-10-01", - "server": "cloudflare", - "set-cookie": "[REDACTED]", - "strict-transport-security": "max-age=31536000; includeSubDomains; preload", - "transfer-encoding": "chunked", - "x-content-type-options": "nosniff", - "x-ratelimit-limit-requests": "30000", - "x-ratelimit-limit-tokens": "180000000", - "x-ratelimit-remaining-requests": "29999", - "x-ratelimit-remaining-tokens": "179998176", - "x-ratelimit-reset-requests": "2ms", - "x-ratelimit-reset-tokens": "0s", - "x-request-id": "req_f5479925a7844533a4ae27ed832d027b" - }, - "status": 200, - "statusText": "OK" - } - }, - { - "callIndex": 10, - "id": "ab34eb26595033a1", - "matchKey": "POST api.openai.com/v1/responses", - "recordedAt": "2026-08-03T09:12:12.905Z", - "request": { - "body": { - "kind": "json", - "value": { - "input": [ - { - "content": "You are running in headless mode with no human operator. Work autonomously — never ask questions, never wait for user input. Make your best judgment and proceed independently.\n\nYou are a deterministic Flue instrumentation test agent. Follow user instructions exactly. When asked for a marker, output only that marker and no extra text. When running a local skill file, read it yourself and do not delegate it to a task.\n\n## Available Skills\n\nThe following skills provide specialized instructions for specific tasks. When a task matches a skill description, activate that skill before proceeding so its full instructions are loaded. Skill instructions and supporting resources stay lazy until activation or explicit file reads.\n\n- **e2e-flue-skill** — Deterministic skill used by the Braintrust Flue instrumentation e2e test.\n\nDate: Mon, Aug 3, 2026\nWorking directory: .\n\nDirectory structure:\n.agents\n.flue\n.flue-build\n__cassettes__\n__snapshots__\nassertions.ts\ncassette-filter.mjs\nconstants.mjs\ndist\nnode_modules\npackage.json\npnpm-lock.yaml\nscenario.cli.mjs\nscenario.impl.mjs\nscenario.mjs\nscenario.test.ts\nscenario.ts\nscenario.v2.ts\nv2-assertions.ts\nversions", - "role": "developer" - }, - { - "content": [ - { - "text": "Run the skill named \"e2e-flue-skill\" with arguments {\"marker\":\"SKILL_DONE\"}. Output only the marker and no extra text when finished.", - "type": "input_text" - } - ], - "role": "user" - }, - { - "arguments": "{\"description\":\"Run e2e-flue-skill with marker SKILL_DONE\",\"prompt\":\"Run the skill named \\\"e2e-flue-skill\\\" with arguments {\\\"marker\\\":\\\"SKILL_DONE\\\"}. Ensure completion and report the final output marker only.\",\"agent\":\"e2e-flue-skill\",\"cwd\":\".\"}", - "call_id": "call_w6pyYZgnhlSOAXul75DAG1Cj", - "id": "fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6", - "name": "task", - "type": "function_call" - }, - { - "call_id": "call_w6pyYZgnhlSOAXul75DAG1Cj", - "output": "[flue] Subagent \"e2e-flue-skill\" is not declared. Available: (none).", - "type": "function_call_output" - }, - { - "arguments": "{\"command\":\"find . -maxdepth 4 -name AGENTS.md -print\",\"timeout\":100000}", - "call_id": "call_6zEBnJRmXFIkGFwNjWLZP4uC", - "id": "fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee", - "name": "bash", - "type": "function_call" - }, - { - "call_id": "call_6zEBnJRmXFIkGFwNjWLZP4uC", - "output": "total 4\ndrwxr-xr-x 3 vercel-sandbox vercel-sandbox 20 Aug 3 09:10 .\ndrwxr-xr-x 10 vercel-sandbox vercel-sandbox 4096 Aug 3 09:11 ..\ndrwxr-xr-x 3 vercel-sandbox vercel-sandbox 28 Aug 3 09:10 skills", - "type": "function_call_output" - }, - { - "arguments": "{\"command\":\"find . -maxdepth 4 -type f -name AGENTS.md -print -exec sed -n '1,200p' {} \\\\;\",\"timeout\":100000}", - "call_id": "call_jQUCie0ZgaXUrZCDcezp0msl", - "id": "fc_054a16a2f2ea3217016a705b67575881a3aacde02b9053a321", - "name": "bash", - "type": "function_call" - }, - { - "call_id": "call_jQUCie0ZgaXUrZCDcezp0msl", - "output": "(no output)", - "type": "function_call_output" - }, - { - "arguments": "{\"command\":\"find . -maxdepth 6 -type f -name 'AGENTS.md' -print\",\"timeout\":100000}", - "call_id": "call_NgMSUb6wIR0GAf9U8LXERtBQ", - "id": "fc_054a16a2f2ea3217016a705b68c39881a3b1e518599236e41a", - "name": "bash", - "type": "function_call" - }, - { - "call_id": "call_NgMSUb6wIR0GAf9U8LXERtBQ", - "output": "(no output)", - "type": "function_call_output" - }, - { - "arguments": "{\"command\":\"find . -maxdepth 5 -type f -name '*flue*skill*' -o -name '*skill*' | head\",\"timeout\":100000}", - "call_id": "call_BXKiQ5KG8c42XPPp94zgcY3H", - "id": "fc_054a16a2f2ea3217016a705b69edd081a38c31583ec56eca5f", - "name": "bash", - "type": "function_call" - }, - { - "call_id": "call_BXKiQ5KG8c42XPPp94zgcY3H", - "output": "./.agents/skills\n./.agents/skills/e2e-flue-skill", - "type": "function_call_output" - } - ], - "model": "gpt-5.4-nano", - "prompt_cache_key": "[REDACTED]", - "reasoning": { - "effort": "none" - }, - "store": false, - "stream": true, - "tools": [ - { - "description": "Read a file or list a directory. For files, output is truncated to 2000 lines or 50KB — use offset/limit for large files. For directories, returns the list of entries.", - "name": "read", - "parameters": { - "properties": { - "limit": { - "description": "Maximum number of lines to read", - "type": "number" - }, - "offset": { - "description": "Line number to start from (1-indexed)", - "type": "number" - }, - "path": { - "description": "Path to the file to read", - "type": "string" - } - }, - "required": ["path"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Write content to a file. Creates the file and parent directories if they do not exist.", - "name": "write", - "parameters": { - "properties": { - "content": { - "description": "Content to write to the file", - "type": "string" - }, - "path": { - "description": "Path to the file to write", - "type": "string" - } - }, - "required": ["path", "content"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Edit a file using exact text replacement. The oldText must match a unique region of the file. Use replaceAll to replace all occurrences.", - "name": "edit", - "parameters": { - "properties": { - "newText": { - "description": "Replacement text", - "type": "string" - }, - "oldText": { - "description": "Exact text to find (must be unique)", - "type": "string" - }, - "path": { - "description": "Path to the file to edit", - "type": "string" - }, - "replaceAll": { - "description": "Replace all occurrences", - "type": "boolean" - } - }, - "required": ["path", "oldText", "newText"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Execute a bash command. Returns stdout and stderr. Output is truncated to the last 2000 lines or 50KB.", - "name": "bash", - "parameters": { - "properties": { - "command": { - "description": "Bash command to execute", - "type": "string" - }, - "timeout": { - "description": "Timeout in seconds", - "type": "number" - } - }, - "required": ["command"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Search file contents for a regex pattern. Returns matching lines with file paths and line numbers.", - "name": "grep", - "parameters": { - "properties": { - "include": { - "description": "Glob filter, e.g. \"*.ts\"", - "type": "string" - }, - "path": { - "description": "Directory or file to search (default: .)", - "type": "string" - }, - "pattern": { - "description": "Search pattern (regex)", - "type": "string" - } - }, - "required": ["pattern"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Find files by filename pattern using shell find -name semantics. Returns matching file paths.", - "name": "glob", - "parameters": { - "properties": { - "path": { - "description": "Directory to search in (default: .)", - "type": "string" - }, - "pattern": { - "description": "Filename pattern, e.g. \"*.ts\"", - "type": "string" - } - }, - "required": ["pattern"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Delegate a focused task to a detached child agent with its own context. Use this for independent research, file exploration, or parallel work. The task returns only its final answer to this conversation. No subagents are currently defined.", - "name": "task", - "parameters": { - "properties": { - "agent": { - "description": "Declared subagent to use for the child agent", - "type": "string" - }, - "cwd": { - "description": "Working directory for the child agent. AGENTS.md and skills are discovered from here.", - "type": "string" - }, - "description": { - "description": "Short human-readable label for the delegated work", - "type": "string" - }, - "prompt": { - "description": "Focused instructions for the child agent", - "type": "string" - } - }, - "required": ["prompt"], - "type": "object" - }, - "strict": false, - "type": "function" - } - ] - } - }, - "headers": {}, - "method": "POST", - "url": "https://api.openai.com/v1/responses" - }, - "response": { - "body": { - "chunks": [ - "event: response.created\ndata: {\"type\":\"response.created\",\"response\":{\"id\":\"resp_054a16a2f2ea3217016a705b6af7d081a396a76851300752a1\",\"object\":\"response\",\"created_at\":1785748331,\"status\":\"in_progress\",\"background\":false,\"completed_at\":null,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":null,\"max_tool_calls\":null,\"model\":\"gpt-5.4-nano-2026-03-17\",\"moderation\":null,\"output\":[],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":\"workflow:instrumentation:01KZ3E9TWTAEK4P4P3F63HKAKC::default::ta\",\"prompt_cache_retention\":\"24h\",\"reasoning\":{\"context\":\"current_turn\",\"effort\":\"none\",\"mode\":\"standard\",\"summary\":null},\"safety_identifier\":null,\"service_tier\":\"auto\",\"store\":false,\"temperature\":1,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tool_usage\":{\"image_gen\":{\"input_tokens\":0,\"input_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"output_tokens\":0,\"output_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"total_tokens\":0},\"web_search\":{\"num_requests\":0}},\"tools\":[{\"type\":\"function\",\"description\":\"Read a file or list a directory. For files, output is truncated to 2000 lines or 50KB — use offset/limit for large files. For directories, returns the list of entries.\",\"name\":\"read\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to read\"},\"offset\":{\"type\":\"number\",\"description\":\"Line number to start from (1-indexed)\"},\"limit\":{\"type\":\"number\",\"description\":\"Maximum number of lines to read\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Write content to a file. Creates the file and parent directories if they do not exist.\",\"name\":\"write\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"content\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to write\"},\"content\":{\"type\":\"string\",\"description\":\"Content to write to the file\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Edit a file using exact text replacement. The oldText must match a unique region of the file. Use replaceAll to replace all occurrences.\",\"name\":\"edit\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"oldText\",\"newText\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to edit\"},\"oldText\":{\"type\":\"string\",\"description\":\"Exact text to find (must be unique)\"},\"newText\":{\"type\":\"string\",\"description\":\"Replacement text\"},\"replaceAll\":{\"type\":\"boolean\",\"description\":\"Replace all occurrences\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Execute a bash command. Returns stdout and stderr. Output is truncated to the last 2000 lines or 50KB.\",\"name\":\"bash\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"command\"],\"properties\":{\"command\":{\"type\":\"string\",\"description\":\"Bash command to execute\"},\"timeout\":{\"type\":\"number\",\"description\":\"Timeout in seconds\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Search file contents for a regex pattern. Returns matching lines with file paths and line numbers.\",\"name\":\"grep\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Search pattern (regex)\"},\"path\":{\"type\":\"string\",\"description\":\"Directory or file to search (default: .)\"},\"include\":{\"type\":\"string\",\"description\":\"Glob filter, e.g. \\\"*.ts\\\"\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Find files by filename pattern using shell find -name semantics. Returns matching file paths.\",\"name\":\"glob\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Filename pattern, e.g. \\\"*.ts\\\"\"},\"path\":{\"type\":\"string\",\"description\":\"Directory to search in (default: .)\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Delegate a focused task to a detached child agent with its own context. Use this for independent research, file exploration, or parallel work. The task returns only its final answer to this conversation. No subagents are currently defined.\",\"name\":\"task\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"prompt\"],\"properties\":{\"description\":{\"type\":\"string\",\"description\":\"Short human-readable label for the delegated work\"},\"prompt\":{\"type\":\"string\",\"description\":\"Focused instructions for the child agent\"},\"agent\":{\"type\":\"string\",\"description\":\"Declared subagent to use for the child agent\"},\"cwd\":{\"type\":\"string\",\"description\":\"Working directory for the child agent. AGENTS.md and skills are discovered from here.\"}}},\"strict\":false}],\"top_logprobs\":0,\"top_p\":0.98,\"truncation\":\"disabled\",\"usage\":null,\"user\":null,\"metadata\":{}},\"sequence_number\":0}", - "event: response.in_progress\ndata: {\"type\":\"response.in_progress\",\"response\":{\"id\":\"resp_054a16a2f2ea3217016a705b6af7d081a396a76851300752a1\",\"object\":\"response\",\"created_at\":1785748331,\"status\":\"in_progress\",\"background\":false,\"completed_at\":null,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":null,\"max_tool_calls\":null,\"model\":\"gpt-5.4-nano-2026-03-17\",\"moderation\":null,\"output\":[],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":\"workflow:instrumentation:01KZ3E9TWTAEK4P4P3F63HKAKC::default::ta\",\"prompt_cache_retention\":\"24h\",\"reasoning\":{\"context\":\"current_turn\",\"effort\":\"none\",\"mode\":\"standard\",\"summary\":null},\"safety_identifier\":null,\"service_tier\":\"auto\",\"store\":false,\"temperature\":1,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tool_usage\":{\"image_gen\":{\"input_tokens\":0,\"input_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"output_tokens\":0,\"output_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"total_tokens\":0},\"web_search\":{\"num_requests\":0}},\"tools\":[{\"type\":\"function\",\"description\":\"Read a file or list a directory. For files, output is truncated to 2000 lines or 50KB — use offset/limit for large files. For directories, returns the list of entries.\",\"name\":\"read\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to read\"},\"offset\":{\"type\":\"number\",\"description\":\"Line number to start from (1-indexed)\"},\"limit\":{\"type\":\"number\",\"description\":\"Maximum number of lines to read\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Write content to a file. Creates the file and parent directories if they do not exist.\",\"name\":\"write\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"content\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to write\"},\"content\":{\"type\":\"string\",\"description\":\"Content to write to the file\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Edit a file using exact text replacement. The oldText must match a unique region of the file. Use replaceAll to replace all occurrences.\",\"name\":\"edit\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"oldText\",\"newText\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to edit\"},\"oldText\":{\"type\":\"string\",\"description\":\"Exact text to find (must be unique)\"},\"newText\":{\"type\":\"string\",\"description\":\"Replacement text\"},\"replaceAll\":{\"type\":\"boolean\",\"description\":\"Replace all occurrences\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Execute a bash command. Returns stdout and stderr. Output is truncated to the last 2000 lines or 50KB.\",\"name\":\"bash\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"command\"],\"properties\":{\"command\":{\"type\":\"string\",\"description\":\"Bash command to execute\"},\"timeout\":{\"type\":\"number\",\"description\":\"Timeout in seconds\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Search file contents for a regex pattern. Returns matching lines with file paths and line numbers.\",\"name\":\"grep\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Search pattern (regex)\"},\"path\":{\"type\":\"string\",\"description\":\"Directory or file to search (default: .)\"},\"include\":{\"type\":\"string\",\"description\":\"Glob filter, e.g. \\\"*.ts\\\"\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Find files by filename pattern using shell find -name semantics. Returns matching file paths.\",\"name\":\"glob\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Filename pattern, e.g. \\\"*.ts\\\"\"},\"path\":{\"type\":\"string\",\"description\":\"Directory to search in (default: .)\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Delegate a focused task to a detached child agent with its own context. Use this for independent research, file exploration, or parallel work. The task returns only its final answer to this conversation. No subagents are currently defined.\",\"name\":\"task\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"prompt\"],\"properties\":{\"description\":{\"type\":\"string\",\"description\":\"Short human-readable label for the delegated work\"},\"prompt\":{\"type\":\"string\",\"description\":\"Focused instructions for the child agent\"},\"agent\":{\"type\":\"string\",\"description\":\"Declared subagent to use for the child agent\"},\"cwd\":{\"type\":\"string\",\"description\":\"Working directory for the child agent. AGENTS.md and skills are discovered from here.\"}}},\"strict\":false}],\"top_logprobs\":0,\"top_p\":0.98,\"truncation\":\"disabled\",\"usage\":null,\"user\":null,\"metadata\":{}},\"sequence_number\":1}", - "event: response.output_item.added\ndata: {\"type\":\"response.output_item.added\",\"item\":{\"id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"type\":\"function_call\",\"status\":\"in_progress\",\"arguments\":\"\",\"call_id\":\"call_Zmmrs64LFscpAHsIUaIfEcAN\",\"name\":\"bash\"},\"output_index\":0,\"sequence_number\":2}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"{\\\"\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"K1hVgurfEtUc37\",\"output_index\":0,\"sequence_number\":3}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"command\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"luCO3JGp3\",\"output_index\":0,\"sequence_number\":4}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"\\\":\\\"\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"FmitdgZUwsXVs\",\"output_index\":0,\"sequence_number\":5}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"ls\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"VZpJ9At0woeytk\",\"output_index\":0,\"sequence_number\":6}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" -\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"0xukceLNBwYJiK\",\"output_index\":0,\"sequence_number\":7}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"la\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"1s2AnPNungEZlQ\",\"output_index\":0,\"sequence_number\":8}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" /\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"Bfcx9jEBxq6eit\",\"output_index\":0,\"sequence_number\":9}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"ver\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"1rFsvnpXQsSA2\",\"output_index\":0,\"sequence_number\":10}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"cel\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"WHxut9fzLrXkW\",\"output_index\":0,\"sequence_number\":11}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/s\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"BzqeyKc2WZeyRg\",\"output_index\":0,\"sequence_number\":12}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"andbox\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"ii7jPgkIJ8\",\"output_index\":0,\"sequence_number\":13}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"PkykxLdvlfrkHPL\",\"output_index\":0,\"sequence_number\":14}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"brain\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"12UfuwAycuU\",\"output_index\":0,\"sequence_number\":15}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"trust\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"3K3OjXe08nc\",\"output_index\":0,\"sequence_number\":16}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"data\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"PkaaeYRGj55a\",\"output_index\":0,\"sequence_number\":17}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"UHfCbDbKjO5lYqr\",\"output_index\":0,\"sequence_number\":18}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"brain\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"MOTWOSsU43Z\",\"output_index\":0,\"sequence_number\":19}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"trust\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"dmMQk0sQrGh\",\"output_index\":0,\"sequence_number\":20}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"-sdk\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"XtEiI6ADgEIV\",\"output_index\":0,\"sequence_number\":21}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"-j\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"5ovKtom8zqryPR\",\"output_index\":0,\"sequence_number\":22}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"avascript\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"7eUUx0L\",\"output_index\":0,\"sequence_number\":23}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/e\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"9QbEfSAXVpmE07\",\"output_index\":0,\"sequence_number\":24}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"2\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"BeGJDWOdDgtyZwk\",\"output_index\":0,\"sequence_number\":25}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"e\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"Vio65vRQsUs4hgC\",\"output_index\":0,\"sequence_number\":26}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/sc\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"6P7cD2KzvBQE3\",\"output_index\":0,\"sequence_number\":27}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"enarios\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"RJSR2IM5S\",\"output_index\":0,\"sequence_number\":28}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/fl\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"uSAAuJhHqx2h6\",\"output_index\":0,\"sequence_number\":29}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"ue\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"LURBTZTvJSTf7s\",\"output_index\":0,\"sequence_number\":30}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"-in\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"b3ZtE4ekxYWXs\",\"output_index\":0,\"sequence_number\":31}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"strument\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"WUoTVOo4\",\"output_index\":0,\"sequence_number\":32}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"ation\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"n6TvbFOtdSH\",\"output_index\":0,\"sequence_number\":33}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/.\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"1Lt96k7JA9dUeI\",\"output_index\":0,\"sequence_number\":34}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"agents\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"O7qjKmlEms\",\"output_index\":0,\"sequence_number\":35}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"I1E6t9owIg9P1WR\",\"output_index\":0,\"sequence_number\":36}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"skills\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"21ZpKkvrdb\",\"output_index\":0,\"sequence_number\":37}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/e\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"zV3b4fO0R5USoJ\",\"output_index\":0,\"sequence_number\":38}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"2\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"wtfZeZvShndVNQM\",\"output_index\":0,\"sequence_number\":39}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"e\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"8DgrYaTc5tF9qQN\",\"output_index\":0,\"sequence_number\":40}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"-fl\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"WMhoTEOnE6Xro\",\"output_index\":0,\"sequence_number\":41}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"ue\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"XaFg3AblnqowIQ\",\"output_index\":0,\"sequence_number\":42}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"-s\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"FJ1AtHWIUTZkuK\",\"output_index\":0,\"sequence_number\":43}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"kill\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"K2KXxbd3uIZt\",\"output_index\":0,\"sequence_number\":44}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" &&\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"ek6YUPE8Cp9AE\",\"output_index\":0,\"sequence_number\":45}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" find\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"XM049MlPwBE\",\"output_index\":0,\"sequence_number\":46}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" /\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"SCFb56qc5ogbCE\",\"output_index\":0,\"sequence_number\":47}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"ver\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"nB5JRT4lhiSpZ\",\"output_index\":0,\"sequence_number\":48}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"cel\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"DozjlarmrsGSB\",\"output_index\":0,\"sequence_number\":49}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/s\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"CvaFfLoJcDammS\",\"output_index\":0,\"sequence_number\":50}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"andbox\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"6yYxC29jBg\",\"output_index\":0,\"sequence_number\":51}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"NpUUfvLkXCyoKMW\",\"output_index\":0,\"sequence_number\":52}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"brain\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"ym9bVn0oRVM\",\"output_index\":0,\"sequence_number\":53}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"trust\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"BPomQliEf6i\",\"output_index\":0,\"sequence_number\":54}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"data\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"0hcPCne66Cg8\",\"output_index\":0,\"sequence_number\":55}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"TfdWF7zbYNainuF\",\"output_index\":0,\"sequence_number\":56}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"brain\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"eYEUL1yBJZh\",\"output_index\":0,\"sequence_number\":57}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"trust\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"EfjIxAAX6o0\",\"output_index\":0,\"sequence_number\":58}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"-sdk\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"8hFJ6Wya2ZL8\",\"output_index\":0,\"sequence_number\":59}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"-j\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"BUmLaB9uO7p4JA\",\"output_index\":0,\"sequence_number\":60}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"avascript\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"TZCxxmh\",\"output_index\":0,\"sequence_number\":61}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/e\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"Wkjm95wXRvIPMz\",\"output_index\":0,\"sequence_number\":62}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"2\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"IWO2TynZ5pN8CwA\",\"output_index\":0,\"sequence_number\":63}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"e\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"AYKtiZZuh0gJgYj\",\"output_index\":0,\"sequence_number\":64}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/sc\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"rcDuq1t53ub50\",\"output_index\":0,\"sequence_number\":65}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"enarios\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"cPtht6oAy\",\"output_index\":0,\"sequence_number\":66}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/fl\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"tQudFftzB650V\",\"output_index\":0,\"sequence_number\":67}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"ue\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"3iUp76L3SIYhqb\",\"output_index\":0,\"sequence_number\":68}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"-in\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"ZXAa0ULdLud1i\",\"output_index\":0,\"sequence_number\":69}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"strument\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"kzf0iJxr\",\"output_index\":0,\"sequence_number\":70}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"ation\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"rw7Suy9RUTq\",\"output_index\":0,\"sequence_number\":71}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/.\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"uPuc49BoOe4rwv\",\"output_index\":0,\"sequence_number\":72}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"agents\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"xCcbSL9wCG\",\"output_index\":0,\"sequence_number\":73}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"0kPSap2qJXeYe5c\",\"output_index\":0,\"sequence_number\":74}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"skills\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"07SeHSKAgE\",\"output_index\":0,\"sequence_number\":75}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/e\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"g2UQCFv27V3sqv\",\"output_index\":0,\"sequence_number\":76}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"2\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"YniEsIzqh6veIWP\",\"output_index\":0,\"sequence_number\":77}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"e\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"64iQG5gdZWPuPkQ\",\"output_index\":0,\"sequence_number\":78}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"-fl\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"Kz4PY7cfevVr9\",\"output_index\":0,\"sequence_number\":79}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"ue\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"dAurB55pjAoLFj\",\"output_index\":0,\"sequence_number\":80}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"-s\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"o5PWW9PAQB5Kvq\",\"output_index\":0,\"sequence_number\":81}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"kill\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"kEfcPgdMqlKI\",\"output_index\":0,\"sequence_number\":82}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" -\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"SJWGXR6ryAJ2DE\",\"output_index\":0,\"sequence_number\":83}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"max\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"4QU605ENItN8B\",\"output_index\":0,\"sequence_number\":84}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"depth\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"4XthyjdZM9w\",\"output_index\":0,\"sequence_number\":85}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" \",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"WgGDumDYzUufFUI\",\"output_index\":0,\"sequence_number\":86}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"2\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"oQ8wQwbVorceatB\",\"output_index\":0,\"sequence_number\":87}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" -\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"sxrOiShGGRNzfx\",\"output_index\":0,\"sequence_number\":88}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"type\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"BwSNPMU6iGnd\",\"output_index\":0,\"sequence_number\":89}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" f\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"Rm35RryQU13YdR\",\"output_index\":0,\"sequence_number\":90}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" -\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"zJ5ddKoSwCzVU4\",\"output_index\":0,\"sequence_number\":91}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"print\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"h80D5JBABPH\",\"output_index\":0,\"sequence_number\":92}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"\\\",\\\"\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"rpdDODEBJorbc\",\"output_index\":0,\"sequence_number\":93}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"timeout\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"p4vKlv5m6\",\"output_index\":0,\"sequence_number\":94}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"\\\":\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"Zbhx8YnzpAicDP\",\"output_index\":0,\"sequence_number\":95}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"100\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"r4y6kurWe2zky\",\"output_index\":0,\"sequence_number\":96}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"000\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"x0tuUliZpjxkA\",\"output_index\":0,\"sequence_number\":97}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"}\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"VTLJl2mMhQOUOyQ\",\"output_index\":0,\"sequence_number\":98}", - "event: response.function_call_arguments.done\ndata: {\"type\":\"response.function_call_arguments.done\",\"arguments\":\"{\\\"command\\\":\\\"find ./.agents/skills/e2e-flue-skill -maxdepth 2 -type f -print\\\",\\\"timeout\\\":100000}\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"output_index\":0,\"sequence_number\":99}", - "event: response.output_item.done\ndata: {\"type\":\"response.output_item.done\",\"item\":{\"id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"type\":\"function_call\",\"status\":\"completed\",\"arguments\":\"{\\\"command\\\":\\\"find ./.agents/skills/e2e-flue-skill -maxdepth 2 -type f -print\\\",\\\"timeout\\\":100000}\",\"call_id\":\"call_Zmmrs64LFscpAHsIUaIfEcAN\",\"name\":\"bash\"},\"output_index\":0,\"sequence_number\":100}", - "event: response.completed\ndata: {\"type\":\"response.completed\",\"response\":{\"id\":\"resp_054a16a2f2ea3217016a705b6af7d081a396a76851300752a1\",\"object\":\"response\",\"created_at\":1785748331,\"status\":\"completed\",\"background\":false,\"completed_at\":1785748332,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":null,\"max_tool_calls\":null,\"model\":\"gpt-5.4-nano-2026-03-17\",\"moderation\":null,\"output\":[{\"id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"type\":\"function_call\",\"status\":\"completed\",\"arguments\":\"{\\\"command\\\":\\\"find ./.agents/skills/e2e-flue-skill -maxdepth 2 -type f -print\\\",\\\"timeout\\\":100000}\",\"call_id\":\"call_Zmmrs64LFscpAHsIUaIfEcAN\",\"name\":\"bash\"}],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":\"workflow:instrumentation:01KZ3E9TWTAEK4P4P3F63HKAKC::default::ta\",\"prompt_cache_retention\":\"24h\",\"reasoning\":{\"context\":\"current_turn\",\"effort\":\"none\",\"mode\":\"standard\",\"summary\":null},\"safety_identifier\":null,\"service_tier\":\"default\",\"store\":false,\"temperature\":1,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tool_usage\":{\"image_gen\":{\"input_tokens\":0,\"input_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"output_tokens\":0,\"output_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"total_tokens\":0},\"web_search\":{\"num_requests\":0}},\"tools\":[{\"type\":\"function\",\"description\":\"Read a file or list a directory. For files, output is truncated to 2000 lines or 50KB — use offset/limit for large files. For directories, returns the list of entries.\",\"name\":\"read\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to read\"},\"offset\":{\"type\":\"number\",\"description\":\"Line number to start from (1-indexed)\"},\"limit\":{\"type\":\"number\",\"description\":\"Maximum number of lines to read\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Write content to a file. Creates the file and parent directories if they do not exist.\",\"name\":\"write\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"content\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to write\"},\"content\":{\"type\":\"string\",\"description\":\"Content to write to the file\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Edit a file using exact text replacement. The oldText must match a unique region of the file. Use replaceAll to replace all occurrences.\",\"name\":\"edit\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"oldText\",\"newText\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to edit\"},\"oldText\":{\"type\":\"string\",\"description\":\"Exact text to find (must be unique)\"},\"newText\":{\"type\":\"string\",\"description\":\"Replacement text\"},\"replaceAll\":{\"type\":\"boolean\",\"description\":\"Replace all occurrences\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Execute a bash command. Returns stdout and stderr. Output is truncated to the last 2000 lines or 50KB.\",\"name\":\"bash\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"command\"],\"properties\":{\"command\":{\"type\":\"string\",\"description\":\"Bash command to execute\"},\"timeout\":{\"type\":\"number\",\"description\":\"Timeout in seconds\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Search file contents for a regex pattern. Returns matching lines with file paths and line numbers.\",\"name\":\"grep\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Search pattern (regex)\"},\"path\":{\"type\":\"string\",\"description\":\"Directory or file to search (default: .)\"},\"include\":{\"type\":\"string\",\"description\":\"Glob filter, e.g. \\\"*.ts\\\"\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Find files by filename pattern using shell find -name semantics. Returns matching file paths.\",\"name\":\"glob\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Filename pattern, e.g. \\\"*.ts\\\"\"},\"path\":{\"type\":\"string\",\"description\":\"Directory to search in (default: .)\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Delegate a focused task to a detached child agent with its own context. Use this for independent research, file exploration, or parallel work. The task returns only its final answer to this conversation. No subagents are currently defined.\",\"name\":\"task\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"prompt\"],\"properties\":{\"description\":{\"type\":\"string\",\"description\":\"Short human-readable label for the delegated work\"},\"prompt\":{\"type\":\"string\",\"description\":\"Focused instructions for the child agent\"},\"agent\":{\"type\":\"string\",\"description\":\"Declared subagent to use for the child agent\"},\"cwd\":{\"type\":\"string\",\"description\":\"Working directory for the child agent. AGENTS.md and skills are discovered from here.\"}}},\"strict\":false}],\"top_logprobs\":0,\"top_p\":0.98,\"truncation\":\"disabled\",\"usage\":{\"input_tokens\":1513,\"input_tokens_details\":{\"cache_write_tokens\":0,\"cached_tokens\":0},\"output_tokens\":109,\"output_tokens_details\":{\"reasoning_tokens\":0},\"total_tokens\":1622},\"user\":null,\"metadata\":{}},\"sequence_number\":101}" - ], - "kind": "sse" - }, - "headers": { - "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", - "alt-svc": "h3=\":443\"; ma=86400", - "cf-cache-status": "DYNAMIC", - "cf-ray": "a25432f9bf6f7bb6-IAD", - "connection": "keep-alive", - "content-type": "text/event-stream; charset=utf-8", - "date": "Mon, 03 Aug 2026 09:12:11 GMT", - "openai-organization": "braintrust-data", - "openai-processing-ms": "688", - "openai-project": "proj_vsCSXafhhByzWOThMrJcZiw9", - "openai-version": "2020-10-01", - "server": "cloudflare", - "set-cookie": "[REDACTED]", - "strict-transport-security": "max-age=31536000; includeSubDomains; preload", - "transfer-encoding": "chunked", - "x-content-type-options": "nosniff", - "x-ratelimit-limit-requests": "30000", - "x-ratelimit-limit-tokens": "180000000", - "x-ratelimit-remaining-requests": "29999", - "x-ratelimit-remaining-tokens": "179998032", - "x-ratelimit-reset-requests": "2ms", - "x-ratelimit-reset-tokens": "0s", - "x-request-id": "req_2ceebd83455e4b27b79fc2d72fb5dd3c" - }, - "status": 200, - "statusText": "OK" - } - }, - { - "callIndex": 11, - "id": "250e64f864c72337", - "matchKey": "POST api.openai.com/v1/responses", - "recordedAt": "2026-08-03T09:12:14.154Z", - "request": { - "body": { - "kind": "json", - "value": { - "input": [ - { - "content": "You are running in headless mode with no human operator. Work autonomously — never ask questions, never wait for user input. Make your best judgment and proceed independently.\n\nYou are a deterministic Flue instrumentation test agent. Follow user instructions exactly. When asked for a marker, output only that marker and no extra text. When running a local skill file, read it yourself and do not delegate it to a task.\n\n## Available Skills\n\nThe following skills provide specialized instructions for specific tasks. When a task matches a skill description, activate that skill before proceeding so its full instructions are loaded. Skill instructions and supporting resources stay lazy until activation or explicit file reads.\n\n- **e2e-flue-skill** — Deterministic skill used by the Braintrust Flue instrumentation e2e test.\n\nDate: Mon, Aug 3, 2026\nWorking directory: .\n\nDirectory structure:\n.agents\n.flue\n.flue-build\n__cassettes__\n__snapshots__\nassertions.ts\ncassette-filter.mjs\nconstants.mjs\ndist\nnode_modules\npackage.json\npnpm-lock.yaml\nscenario.cli.mjs\nscenario.impl.mjs\nscenario.mjs\nscenario.test.ts\nscenario.ts\nscenario.v2.ts\nv2-assertions.ts\nversions", - "role": "developer" - }, - { - "content": [ - { - "text": "Run the skill named \"e2e-flue-skill\" with arguments {\"marker\":\"SKILL_DONE\"}. Output only the marker and no extra text when finished.", - "type": "input_text" - } - ], - "role": "user" - }, - { - "arguments": "{\"description\":\"Run e2e-flue-skill with marker SKILL_DONE\",\"prompt\":\"Run the skill named \\\"e2e-flue-skill\\\" with arguments {\\\"marker\\\":\\\"SKILL_DONE\\\"}. Ensure completion and report the final output marker only.\",\"agent\":\"e2e-flue-skill\",\"cwd\":\".\"}", - "call_id": "call_w6pyYZgnhlSOAXul75DAG1Cj", - "id": "fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6", - "name": "task", - "type": "function_call" - }, - { - "call_id": "call_w6pyYZgnhlSOAXul75DAG1Cj", - "output": "[flue] Subagent \"e2e-flue-skill\" is not declared. Available: (none).", - "type": "function_call_output" - }, - { - "arguments": "{\"command\":\"find . -maxdepth 4 -name AGENTS.md -print\",\"timeout\":100000}", - "call_id": "call_6zEBnJRmXFIkGFwNjWLZP4uC", - "id": "fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee", - "name": "bash", - "type": "function_call" - }, - { - "call_id": "call_6zEBnJRmXFIkGFwNjWLZP4uC", - "output": "total 4\ndrwxr-xr-x 3 vercel-sandbox vercel-sandbox 20 Aug 3 09:10 .\ndrwxr-xr-x 10 vercel-sandbox vercel-sandbox 4096 Aug 3 09:11 ..\ndrwxr-xr-x 3 vercel-sandbox vercel-sandbox 28 Aug 3 09:10 skills", - "type": "function_call_output" - }, - { - "arguments": "{\"command\":\"find . -maxdepth 4 -type f -name AGENTS.md -print -exec sed -n '1,200p' {} \\\\;\",\"timeout\":100000}", - "call_id": "call_jQUCie0ZgaXUrZCDcezp0msl", - "id": "fc_054a16a2f2ea3217016a705b67575881a3aacde02b9053a321", - "name": "bash", - "type": "function_call" - }, - { - "call_id": "call_jQUCie0ZgaXUrZCDcezp0msl", - "output": "(no output)", - "type": "function_call_output" - }, - { - "arguments": "{\"command\":\"find . -maxdepth 6 -type f -name 'AGENTS.md' -print\",\"timeout\":100000}", - "call_id": "call_NgMSUb6wIR0GAf9U8LXERtBQ", - "id": "fc_054a16a2f2ea3217016a705b68c39881a3b1e518599236e41a", - "name": "bash", - "type": "function_call" - }, - { - "call_id": "call_NgMSUb6wIR0GAf9U8LXERtBQ", - "output": "(no output)", - "type": "function_call_output" - }, - { - "arguments": "{\"command\":\"find . -maxdepth 5 -type f -name '*flue*skill*' -o -name '*skill*' | head\",\"timeout\":100000}", - "call_id": "call_BXKiQ5KG8c42XPPp94zgcY3H", - "id": "fc_054a16a2f2ea3217016a705b69edd081a38c31583ec56eca5f", - "name": "bash", - "type": "function_call" - }, - { - "call_id": "call_BXKiQ5KG8c42XPPp94zgcY3H", - "output": "./.agents/skills\n./.agents/skills/e2e-flue-skill", - "type": "function_call_output" - }, - { - "arguments": "{\"command\":\"find ./.agents/skills/e2e-flue-skill -maxdepth 2 -type f -print\",\"timeout\":100000}", - "call_id": "call_Zmmrs64LFscpAHsIUaIfEcAN", - "id": "fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63", - "name": "bash", - "type": "function_call" - }, - { - "call_id": "call_Zmmrs64LFscpAHsIUaIfEcAN", - "output": "total 4\ndrwxr-xr-x 2 vercel-sandbox vercel-sandbox 22 Aug 3 07:55 .\ndrwxr-xr-x 3 vercel-sandbox vercel-sandbox 28 Aug 3 07:55 ..\n-rw-r--r-- 1 vercel-sandbox vercel-sandbox 194 Aug 3 07:55 SKILL.md\n./.agents/skills/e2e-flue-skill/SKILL.md", - "type": "function_call_output" - } - ], - "model": "gpt-5.4-nano", - "prompt_cache_key": "[REDACTED]", - "reasoning": { - "effort": "none" - }, - "store": false, - "stream": true, - "tools": [ - { - "description": "Read a file or list a directory. For files, output is truncated to 2000 lines or 50KB — use offset/limit for large files. For directories, returns the list of entries.", - "name": "read", - "parameters": { - "properties": { - "limit": { - "description": "Maximum number of lines to read", - "type": "number" - }, - "offset": { - "description": "Line number to start from (1-indexed)", - "type": "number" - }, - "path": { - "description": "Path to the file to read", - "type": "string" - } - }, - "required": ["path"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Write content to a file. Creates the file and parent directories if they do not exist.", - "name": "write", - "parameters": { - "properties": { - "content": { - "description": "Content to write to the file", - "type": "string" - }, - "path": { - "description": "Path to the file to write", - "type": "string" - } - }, - "required": ["path", "content"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Edit a file using exact text replacement. The oldText must match a unique region of the file. Use replaceAll to replace all occurrences.", - "name": "edit", - "parameters": { - "properties": { - "newText": { - "description": "Replacement text", - "type": "string" - }, - "oldText": { - "description": "Exact text to find (must be unique)", - "type": "string" - }, - "path": { - "description": "Path to the file to edit", - "type": "string" - }, - "replaceAll": { - "description": "Replace all occurrences", - "type": "boolean" - } - }, - "required": ["path", "oldText", "newText"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Execute a bash command. Returns stdout and stderr. Output is truncated to the last 2000 lines or 50KB.", - "name": "bash", - "parameters": { - "properties": { - "command": { - "description": "Bash command to execute", - "type": "string" - }, - "timeout": { - "description": "Timeout in seconds", - "type": "number" - } - }, - "required": ["command"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Search file contents for a regex pattern. Returns matching lines with file paths and line numbers.", - "name": "grep", - "parameters": { - "properties": { - "include": { - "description": "Glob filter, e.g. \"*.ts\"", - "type": "string" - }, - "path": { - "description": "Directory or file to search (default: .)", - "type": "string" - }, - "pattern": { - "description": "Search pattern (regex)", - "type": "string" - } - }, - "required": ["pattern"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Find files by filename pattern using shell find -name semantics. Returns matching file paths.", - "name": "glob", - "parameters": { - "properties": { - "path": { - "description": "Directory to search in (default: .)", - "type": "string" - }, - "pattern": { - "description": "Filename pattern, e.g. \"*.ts\"", - "type": "string" - } - }, - "required": ["pattern"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Delegate a focused task to a detached child agent with its own context. Use this for independent research, file exploration, or parallel work. The task returns only its final answer to this conversation. No subagents are currently defined.", - "name": "task", - "parameters": { - "properties": { - "agent": { - "description": "Declared subagent to use for the child agent", - "type": "string" - }, - "cwd": { - "description": "Working directory for the child agent. AGENTS.md and skills are discovered from here.", - "type": "string" - }, - "description": { - "description": "Short human-readable label for the delegated work", - "type": "string" - }, - "prompt": { - "description": "Focused instructions for the child agent", - "type": "string" - } - }, - "required": ["prompt"], - "type": "object" - }, - "strict": false, - "type": "function" - } - ] - } - }, - "headers": {}, - "method": "POST", - "url": "https://api.openai.com/v1/responses" - }, - "response": { - "body": { - "chunks": [ - "event: response.created\ndata: {\"type\":\"response.created\",\"response\":{\"id\":\"resp_054a16a2f2ea3217016a705b6d008c81a3a332b5376350158b\",\"object\":\"response\",\"created_at\":1785748333,\"status\":\"in_progress\",\"background\":false,\"completed_at\":null,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":null,\"max_tool_calls\":null,\"model\":\"gpt-5.4-nano-2026-03-17\",\"moderation\":null,\"output\":[],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":\"workflow:instrumentation:01KZ3E9TWTAEK4P4P3F63HKAKC::default::ta\",\"prompt_cache_retention\":\"24h\",\"reasoning\":{\"context\":\"current_turn\",\"effort\":\"none\",\"mode\":\"standard\",\"summary\":null},\"safety_identifier\":null,\"service_tier\":\"auto\",\"store\":false,\"temperature\":1,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tool_usage\":{\"image_gen\":{\"input_tokens\":0,\"input_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"output_tokens\":0,\"output_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"total_tokens\":0},\"web_search\":{\"num_requests\":0}},\"tools\":[{\"type\":\"function\",\"description\":\"Read a file or list a directory. For files, output is truncated to 2000 lines or 50KB — use offset/limit for large files. For directories, returns the list of entries.\",\"name\":\"read\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to read\"},\"offset\":{\"type\":\"number\",\"description\":\"Line number to start from (1-indexed)\"},\"limit\":{\"type\":\"number\",\"description\":\"Maximum number of lines to read\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Write content to a file. Creates the file and parent directories if they do not exist.\",\"name\":\"write\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"content\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to write\"},\"content\":{\"type\":\"string\",\"description\":\"Content to write to the file\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Edit a file using exact text replacement. The oldText must match a unique region of the file. Use replaceAll to replace all occurrences.\",\"name\":\"edit\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"oldText\",\"newText\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to edit\"},\"oldText\":{\"type\":\"string\",\"description\":\"Exact text to find (must be unique)\"},\"newText\":{\"type\":\"string\",\"description\":\"Replacement text\"},\"replaceAll\":{\"type\":\"boolean\",\"description\":\"Replace all occurrences\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Execute a bash command. Returns stdout and stderr. Output is truncated to the last 2000 lines or 50KB.\",\"name\":\"bash\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"command\"],\"properties\":{\"command\":{\"type\":\"string\",\"description\":\"Bash command to execute\"},\"timeout\":{\"type\":\"number\",\"description\":\"Timeout in seconds\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Search file contents for a regex pattern. Returns matching lines with file paths and line numbers.\",\"name\":\"grep\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Search pattern (regex)\"},\"path\":{\"type\":\"string\",\"description\":\"Directory or file to search (default: .)\"},\"include\":{\"type\":\"string\",\"description\":\"Glob filter, e.g. \\\"*.ts\\\"\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Find files by filename pattern using shell find -name semantics. Returns matching file paths.\",\"name\":\"glob\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Filename pattern, e.g. \\\"*.ts\\\"\"},\"path\":{\"type\":\"string\",\"description\":\"Directory to search in (default: .)\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Delegate a focused task to a detached child agent with its own context. Use this for independent research, file exploration, or parallel work. The task returns only its final answer to this conversation. No subagents are currently defined.\",\"name\":\"task\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"prompt\"],\"properties\":{\"description\":{\"type\":\"string\",\"description\":\"Short human-readable label for the delegated work\"},\"prompt\":{\"type\":\"string\",\"description\":\"Focused instructions for the child agent\"},\"agent\":{\"type\":\"string\",\"description\":\"Declared subagent to use for the child agent\"},\"cwd\":{\"type\":\"string\",\"description\":\"Working directory for the child agent. AGENTS.md and skills are discovered from here.\"}}},\"strict\":false}],\"top_logprobs\":0,\"top_p\":0.98,\"truncation\":\"disabled\",\"usage\":null,\"user\":null,\"metadata\":{}},\"sequence_number\":0}", - "event: response.in_progress\ndata: {\"type\":\"response.in_progress\",\"response\":{\"id\":\"resp_054a16a2f2ea3217016a705b6d008c81a3a332b5376350158b\",\"object\":\"response\",\"created_at\":1785748333,\"status\":\"in_progress\",\"background\":false,\"completed_at\":null,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":null,\"max_tool_calls\":null,\"model\":\"gpt-5.4-nano-2026-03-17\",\"moderation\":null,\"output\":[],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":\"workflow:instrumentation:01KZ3E9TWTAEK4P4P3F63HKAKC::default::ta\",\"prompt_cache_retention\":\"24h\",\"reasoning\":{\"context\":\"current_turn\",\"effort\":\"none\",\"mode\":\"standard\",\"summary\":null},\"safety_identifier\":null,\"service_tier\":\"auto\",\"store\":false,\"temperature\":1,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tool_usage\":{\"image_gen\":{\"input_tokens\":0,\"input_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"output_tokens\":0,\"output_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"total_tokens\":0},\"web_search\":{\"num_requests\":0}},\"tools\":[{\"type\":\"function\",\"description\":\"Read a file or list a directory. For files, output is truncated to 2000 lines or 50KB — use offset/limit for large files. For directories, returns the list of entries.\",\"name\":\"read\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to read\"},\"offset\":{\"type\":\"number\",\"description\":\"Line number to start from (1-indexed)\"},\"limit\":{\"type\":\"number\",\"description\":\"Maximum number of lines to read\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Write content to a file. Creates the file and parent directories if they do not exist.\",\"name\":\"write\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"content\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to write\"},\"content\":{\"type\":\"string\",\"description\":\"Content to write to the file\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Edit a file using exact text replacement. The oldText must match a unique region of the file. Use replaceAll to replace all occurrences.\",\"name\":\"edit\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"oldText\",\"newText\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to edit\"},\"oldText\":{\"type\":\"string\",\"description\":\"Exact text to find (must be unique)\"},\"newText\":{\"type\":\"string\",\"description\":\"Replacement text\"},\"replaceAll\":{\"type\":\"boolean\",\"description\":\"Replace all occurrences\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Execute a bash command. Returns stdout and stderr. Output is truncated to the last 2000 lines or 50KB.\",\"name\":\"bash\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"command\"],\"properties\":{\"command\":{\"type\":\"string\",\"description\":\"Bash command to execute\"},\"timeout\":{\"type\":\"number\",\"description\":\"Timeout in seconds\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Search file contents for a regex pattern. Returns matching lines with file paths and line numbers.\",\"name\":\"grep\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Search pattern (regex)\"},\"path\":{\"type\":\"string\",\"description\":\"Directory or file to search (default: .)\"},\"include\":{\"type\":\"string\",\"description\":\"Glob filter, e.g. \\\"*.ts\\\"\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Find files by filename pattern using shell find -name semantics. Returns matching file paths.\",\"name\":\"glob\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Filename pattern, e.g. \\\"*.ts\\\"\"},\"path\":{\"type\":\"string\",\"description\":\"Directory to search in (default: .)\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Delegate a focused task to a detached child agent with its own context. Use this for independent research, file exploration, or parallel work. The task returns only its final answer to this conversation. No subagents are currently defined.\",\"name\":\"task\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"prompt\"],\"properties\":{\"description\":{\"type\":\"string\",\"description\":\"Short human-readable label for the delegated work\"},\"prompt\":{\"type\":\"string\",\"description\":\"Focused instructions for the child agent\"},\"agent\":{\"type\":\"string\",\"description\":\"Declared subagent to use for the child agent\"},\"cwd\":{\"type\":\"string\",\"description\":\"Working directory for the child agent. AGENTS.md and skills are discovered from here.\"}}},\"strict\":false}],\"top_logprobs\":0,\"top_p\":0.98,\"truncation\":\"disabled\",\"usage\":null,\"user\":null,\"metadata\":{}},\"sequence_number\":1}", - "event: response.output_item.added\ndata: {\"type\":\"response.output_item.added\",\"item\":{\"id\":\"fc_054a16a2f2ea3217016a705b6d71cc81a38ac06634a309a997\",\"type\":\"function_call\",\"status\":\"in_progress\",\"arguments\":\"\",\"call_id\":\"call_KXcpmIy4Cbfugk4SNT2Ge7sG\",\"name\":\"read\"},\"output_index\":0,\"sequence_number\":2}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"{\\\"\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6d71cc81a38ac06634a309a997\",\"obfuscation\":\"AplT1bUEIbLxZO\",\"output_index\":0,\"sequence_number\":3}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"path\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6d71cc81a38ac06634a309a997\",\"obfuscation\":\"m2vm6uJH1fpp\",\"output_index\":0,\"sequence_number\":4}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"\\\":\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6d71cc81a38ac06634a309a997\",\"obfuscation\":\"UE4B3xJGoZjFiD\",\"output_index\":0,\"sequence_number\":5}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"\\\"/\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6d71cc81a38ac06634a309a997\",\"obfuscation\":\"vXAYv4q4OGZeBm\",\"output_index\":0,\"sequence_number\":6}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"ver\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6d71cc81a38ac06634a309a997\",\"obfuscation\":\"lZr7JtLSOpK81\",\"output_index\":0,\"sequence_number\":7}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"cel\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6d71cc81a38ac06634a309a997\",\"obfuscation\":\"AlOhsp6mt6WsL\",\"output_index\":0,\"sequence_number\":8}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/s\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6d71cc81a38ac06634a309a997\",\"obfuscation\":\"5nqW1Lb0h9tswT\",\"output_index\":0,\"sequence_number\":9}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"andbox\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6d71cc81a38ac06634a309a997\",\"obfuscation\":\"A7di1P36tJ\",\"output_index\":0,\"sequence_number\":10}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6d71cc81a38ac06634a309a997\",\"obfuscation\":\"sXrATqCJ4vWOD1I\",\"output_index\":0,\"sequence_number\":11}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"brain\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6d71cc81a38ac06634a309a997\",\"obfuscation\":\"dOlnaweYHcH\",\"output_index\":0,\"sequence_number\":12}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"trust\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6d71cc81a38ac06634a309a997\",\"obfuscation\":\"1e9JdY1CZrh\",\"output_index\":0,\"sequence_number\":13}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"data\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6d71cc81a38ac06634a309a997\",\"obfuscation\":\"ItoYfCrGd190\",\"output_index\":0,\"sequence_number\":14}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6d71cc81a38ac06634a309a997\",\"obfuscation\":\"U2GDfehNMcQdTxK\",\"output_index\":0,\"sequence_number\":15}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"brain\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6d71cc81a38ac06634a309a997\",\"obfuscation\":\"3sMRftpfAWB\",\"output_index\":0,\"sequence_number\":16}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"trust\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6d71cc81a38ac06634a309a997\",\"obfuscation\":\"RtP3uejwoaW\",\"output_index\":0,\"sequence_number\":17}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"-sdk\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6d71cc81a38ac06634a309a997\",\"obfuscation\":\"WTDkaRqKEdlO\",\"output_index\":0,\"sequence_number\":18}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"-j\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6d71cc81a38ac06634a309a997\",\"obfuscation\":\"Sicm3wGutAJOUR\",\"output_index\":0,\"sequence_number\":19}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"avascript\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6d71cc81a38ac06634a309a997\",\"obfuscation\":\"i3Iuw9L\",\"output_index\":0,\"sequence_number\":20}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/e\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6d71cc81a38ac06634a309a997\",\"obfuscation\":\"cNyMn1ymq3OidT\",\"output_index\":0,\"sequence_number\":21}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"2\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6d71cc81a38ac06634a309a997\",\"obfuscation\":\"TF4UthkOjNccn7Q\",\"output_index\":0,\"sequence_number\":22}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"e\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6d71cc81a38ac06634a309a997\",\"obfuscation\":\"WCH0Nr5syDCGWhy\",\"output_index\":0,\"sequence_number\":23}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/sc\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6d71cc81a38ac06634a309a997\",\"obfuscation\":\"9QxeDK80jR88i\",\"output_index\":0,\"sequence_number\":24}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"enarios\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6d71cc81a38ac06634a309a997\",\"obfuscation\":\"ZWEwRV5Aa\",\"output_index\":0,\"sequence_number\":25}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/fl\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6d71cc81a38ac06634a309a997\",\"obfuscation\":\"GCLbXJEBfam0u\",\"output_index\":0,\"sequence_number\":26}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"ue\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6d71cc81a38ac06634a309a997\",\"obfuscation\":\"fYviPD7BD1YoJ7\",\"output_index\":0,\"sequence_number\":27}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"-in\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6d71cc81a38ac06634a309a997\",\"obfuscation\":\"12G2xIB4r4pGx\",\"output_index\":0,\"sequence_number\":28}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"strument\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6d71cc81a38ac06634a309a997\",\"obfuscation\":\"RBo7r0Kw\",\"output_index\":0,\"sequence_number\":29}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"ation\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6d71cc81a38ac06634a309a997\",\"obfuscation\":\"LdN0iq6VGNg\",\"output_index\":0,\"sequence_number\":30}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/.\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6d71cc81a38ac06634a309a997\",\"obfuscation\":\"SC3tJji0aA9vxI\",\"output_index\":0,\"sequence_number\":31}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"agents\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6d71cc81a38ac06634a309a997\",\"obfuscation\":\"46FVinGMH1\",\"output_index\":0,\"sequence_number\":32}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6d71cc81a38ac06634a309a997\",\"obfuscation\":\"Tu4YeGAIlzJv7RO\",\"output_index\":0,\"sequence_number\":33}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"skills\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6d71cc81a38ac06634a309a997\",\"obfuscation\":\"chyXB4Vg5n\",\"output_index\":0,\"sequence_number\":34}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/e\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6d71cc81a38ac06634a309a997\",\"obfuscation\":\"koR43IjON9fzgj\",\"output_index\":0,\"sequence_number\":35}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"2\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6d71cc81a38ac06634a309a997\",\"obfuscation\":\"2PamefwKxxhd235\",\"output_index\":0,\"sequence_number\":36}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"e\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6d71cc81a38ac06634a309a997\",\"obfuscation\":\"7OGNeIaOwukKH4k\",\"output_index\":0,\"sequence_number\":37}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"-fl\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6d71cc81a38ac06634a309a997\",\"obfuscation\":\"LyzjvJ7oVbrNO\",\"output_index\":0,\"sequence_number\":38}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"ue\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6d71cc81a38ac06634a309a997\",\"obfuscation\":\"F6FvzwFZ7LJK05\",\"output_index\":0,\"sequence_number\":39}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"-s\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6d71cc81a38ac06634a309a997\",\"obfuscation\":\"PvR9f24G2kgLxp\",\"output_index\":0,\"sequence_number\":40}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"kill\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6d71cc81a38ac06634a309a997\",\"obfuscation\":\"Cc4f9PrTB9gy\",\"output_index\":0,\"sequence_number\":41}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/S\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6d71cc81a38ac06634a309a997\",\"obfuscation\":\"NFmNKyCIgJRpgX\",\"output_index\":0,\"sequence_number\":42}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"K\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6d71cc81a38ac06634a309a997\",\"obfuscation\":\"i6Hkkg289FwqvEa\",\"output_index\":0,\"sequence_number\":43}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"ILL\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6d71cc81a38ac06634a309a997\",\"obfuscation\":\"Z0w7G2ePBPqRw\",\"output_index\":0,\"sequence_number\":44}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\".md\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6d71cc81a38ac06634a309a997\",\"obfuscation\":\"U8sCPOgqSHwmL\",\"output_index\":0,\"sequence_number\":45}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"\\\",\\\"\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6d71cc81a38ac06634a309a997\",\"obfuscation\":\"Rj1PgiaJnVY9e\",\"output_index\":0,\"sequence_number\":46}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"offset\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6d71cc81a38ac06634a309a997\",\"obfuscation\":\"Nv7VtoPO8D\",\"output_index\":0,\"sequence_number\":47}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"\\\":\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6d71cc81a38ac06634a309a997\",\"obfuscation\":\"xMA7iDRhGXyoRS\",\"output_index\":0,\"sequence_number\":48}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"1\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6d71cc81a38ac06634a309a997\",\"obfuscation\":\"4HmQwBvniK7325Y\",\"output_index\":0,\"sequence_number\":49}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\",\\\"\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6d71cc81a38ac06634a309a997\",\"obfuscation\":\"fp7VCNlkmETdHR\",\"output_index\":0,\"sequence_number\":50}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"limit\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6d71cc81a38ac06634a309a997\",\"obfuscation\":\"JnAHVomsUGV\",\"output_index\":0,\"sequence_number\":51}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"\\\":\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6d71cc81a38ac06634a309a997\",\"obfuscation\":\"oMhJTWBoaVQje5\",\"output_index\":0,\"sequence_number\":52}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"200\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6d71cc81a38ac06634a309a997\",\"obfuscation\":\"pck09M2RCSfdF\",\"output_index\":0,\"sequence_number\":53}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"}\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6d71cc81a38ac06634a309a997\",\"obfuscation\":\"v0WZruprCkbMb5J\",\"output_index\":0,\"sequence_number\":54}", - "event: response.function_call_arguments.done\ndata: {\"type\":\"response.function_call_arguments.done\",\"arguments\":\"{\\\"path\\\":\\\"./.agents/skills/e2e-flue-skill/SKILL.md\\\",\\\"offset\\\":1,\\\"limit\\\":200}\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6d71cc81a38ac06634a309a997\",\"output_index\":0,\"sequence_number\":55}", - "event: response.output_item.done\ndata: {\"type\":\"response.output_item.done\",\"item\":{\"id\":\"fc_054a16a2f2ea3217016a705b6d71cc81a38ac06634a309a997\",\"type\":\"function_call\",\"status\":\"completed\",\"arguments\":\"{\\\"path\\\":\\\"./.agents/skills/e2e-flue-skill/SKILL.md\\\",\\\"offset\\\":1,\\\"limit\\\":200}\",\"call_id\":\"call_KXcpmIy4Cbfugk4SNT2Ge7sG\",\"name\":\"read\"},\"output_index\":0,\"sequence_number\":56}", - "event: response.completed\ndata: {\"type\":\"response.completed\",\"response\":{\"id\":\"resp_054a16a2f2ea3217016a705b6d008c81a3a332b5376350158b\",\"object\":\"response\",\"created_at\":1785748333,\"status\":\"completed\",\"background\":false,\"completed_at\":1785748334,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":null,\"max_tool_calls\":null,\"model\":\"gpt-5.4-nano-2026-03-17\",\"moderation\":null,\"output\":[{\"id\":\"fc_054a16a2f2ea3217016a705b6d71cc81a38ac06634a309a997\",\"type\":\"function_call\",\"status\":\"completed\",\"arguments\":\"{\\\"path\\\":\\\"./.agents/skills/e2e-flue-skill/SKILL.md\\\",\\\"offset\\\":1,\\\"limit\\\":200}\",\"call_id\":\"call_KXcpmIy4Cbfugk4SNT2Ge7sG\",\"name\":\"read\"}],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":\"workflow:instrumentation:01KZ3E9TWTAEK4P4P3F63HKAKC::default::ta\",\"prompt_cache_retention\":\"24h\",\"reasoning\":{\"context\":\"current_turn\",\"effort\":\"none\",\"mode\":\"standard\",\"summary\":null},\"safety_identifier\":null,\"service_tier\":\"default\",\"store\":false,\"temperature\":1,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tool_usage\":{\"image_gen\":{\"input_tokens\":0,\"input_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"output_tokens\":0,\"output_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"total_tokens\":0},\"web_search\":{\"num_requests\":0}},\"tools\":[{\"type\":\"function\",\"description\":\"Read a file or list a directory. For files, output is truncated to 2000 lines or 50KB — use offset/limit for large files. For directories, returns the list of entries.\",\"name\":\"read\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to read\"},\"offset\":{\"type\":\"number\",\"description\":\"Line number to start from (1-indexed)\"},\"limit\":{\"type\":\"number\",\"description\":\"Maximum number of lines to read\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Write content to a file. Creates the file and parent directories if they do not exist.\",\"name\":\"write\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"content\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to write\"},\"content\":{\"type\":\"string\",\"description\":\"Content to write to the file\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Edit a file using exact text replacement. The oldText must match a unique region of the file. Use replaceAll to replace all occurrences.\",\"name\":\"edit\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"oldText\",\"newText\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to edit\"},\"oldText\":{\"type\":\"string\",\"description\":\"Exact text to find (must be unique)\"},\"newText\":{\"type\":\"string\",\"description\":\"Replacement text\"},\"replaceAll\":{\"type\":\"boolean\",\"description\":\"Replace all occurrences\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Execute a bash command. Returns stdout and stderr. Output is truncated to the last 2000 lines or 50KB.\",\"name\":\"bash\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"command\"],\"properties\":{\"command\":{\"type\":\"string\",\"description\":\"Bash command to execute\"},\"timeout\":{\"type\":\"number\",\"description\":\"Timeout in seconds\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Search file contents for a regex pattern. Returns matching lines with file paths and line numbers.\",\"name\":\"grep\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Search pattern (regex)\"},\"path\":{\"type\":\"string\",\"description\":\"Directory or file to search (default: .)\"},\"include\":{\"type\":\"string\",\"description\":\"Glob filter, e.g. \\\"*.ts\\\"\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Find files by filename pattern using shell find -name semantics. Returns matching file paths.\",\"name\":\"glob\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Filename pattern, e.g. \\\"*.ts\\\"\"},\"path\":{\"type\":\"string\",\"description\":\"Directory to search in (default: .)\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Delegate a focused task to a detached child agent with its own context. Use this for independent research, file exploration, or parallel work. The task returns only its final answer to this conversation. No subagents are currently defined.\",\"name\":\"task\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"prompt\"],\"properties\":{\"description\":{\"type\":\"string\",\"description\":\"Short human-readable label for the delegated work\"},\"prompt\":{\"type\":\"string\",\"description\":\"Focused instructions for the child agent\"},\"agent\":{\"type\":\"string\",\"description\":\"Declared subagent to use for the child agent\"},\"cwd\":{\"type\":\"string\",\"description\":\"Working directory for the child agent. AGENTS.md and skills are discovered from here.\"}}},\"strict\":false}],\"top_logprobs\":0,\"top_p\":0.98,\"truncation\":\"disabled\",\"usage\":{\"input_tokens\":1762,\"input_tokens_details\":{\"cache_write_tokens\":0,\"cached_tokens\":1536},\"output_tokens\":64,\"output_tokens_details\":{\"reasoning_tokens\":0},\"total_tokens\":1826},\"user\":null,\"metadata\":{}},\"sequence_number\":57}" - ], - "kind": "sse" - }, - "headers": { - "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", - "alt-svc": "h3=\":443\"; ma=86400", - "cf-cache-status": "DYNAMIC", - "cf-ray": "a2543308de867bb6-IAD", - "connection": "keep-alive", - "content-type": "text/event-stream; charset=utf-8", - "date": "Mon, 03 Aug 2026 09:12:13 GMT", - "openai-organization": "braintrust-data", - "openai-processing-ms": "248", - "openai-project": "proj_vsCSXafhhByzWOThMrJcZiw9", - "openai-version": "2020-10-01", - "server": "cloudflare", - "set-cookie": "[REDACTED]", - "strict-transport-security": "max-age=31536000; includeSubDomains; preload", - "transfer-encoding": "chunked", - "x-content-type-options": "nosniff", - "x-ratelimit-limit-requests": "30000", - "x-ratelimit-limit-tokens": "180000000", - "x-ratelimit-remaining-requests": "29999", - "x-ratelimit-remaining-tokens": "179997783", - "x-ratelimit-reset-requests": "2ms", - "x-ratelimit-reset-tokens": "0s", - "x-request-id": "req_33374d00629445b4aa73562f25cf18f0" - }, - "status": 200, - "statusText": "OK" - } - }, - { - "callIndex": 12, - "id": "fecb7c9f8d04e1bd", - "matchKey": "POST api.openai.com/v1/responses", - "recordedAt": "2026-08-03T09:12:15.213Z", - "request": { - "body": { - "kind": "json", - "value": { - "input": [ - { - "content": "You are running in headless mode with no human operator. Work autonomously — never ask questions, never wait for user input. Make your best judgment and proceed independently.\n\nYou are a deterministic Flue instrumentation test agent. Follow user instructions exactly. When asked for a marker, output only that marker and no extra text. When running a local skill file, read it yourself and do not delegate it to a task.\n\n## Available Skills\n\nThe following skills provide specialized instructions for specific tasks. When a task matches a skill description, activate that skill before proceeding so its full instructions are loaded. Skill instructions and supporting resources stay lazy until activation or explicit file reads.\n\n- **e2e-flue-skill** — Deterministic skill used by the Braintrust Flue instrumentation e2e test.\n\nDate: Mon, Aug 3, 2026\nWorking directory: .\n\nDirectory structure:\n.agents\n.flue\n.flue-build\n__cassettes__\n__snapshots__\nassertions.ts\ncassette-filter.mjs\nconstants.mjs\ndist\nnode_modules\npackage.json\npnpm-lock.yaml\nscenario.cli.mjs\nscenario.impl.mjs\nscenario.mjs\nscenario.test.ts\nscenario.ts\nscenario.v2.ts\nv2-assertions.ts\nversions", - "role": "developer" - }, - { - "content": [ - { - "text": "Run the skill named \"e2e-flue-skill\" with arguments {\"marker\":\"SKILL_DONE\"}. Output only the marker and no extra text when finished.", - "type": "input_text" - } - ], - "role": "user" - }, - { - "arguments": "{\"description\":\"Run e2e-flue-skill with marker SKILL_DONE\",\"prompt\":\"Run the skill named \\\"e2e-flue-skill\\\" with arguments {\\\"marker\\\":\\\"SKILL_DONE\\\"}. Ensure completion and report the final output marker only.\",\"agent\":\"e2e-flue-skill\",\"cwd\":\".\"}", - "call_id": "call_w6pyYZgnhlSOAXul75DAG1Cj", - "id": "fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6", - "name": "task", - "type": "function_call" - }, - { - "call_id": "call_w6pyYZgnhlSOAXul75DAG1Cj", - "output": "[flue] Subagent \"e2e-flue-skill\" is not declared. Available: (none).", - "type": "function_call_output" - }, - { - "arguments": "{\"command\":\"find . -maxdepth 4 -name AGENTS.md -print\",\"timeout\":100000}", - "call_id": "call_6zEBnJRmXFIkGFwNjWLZP4uC", - "id": "fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee", - "name": "bash", - "type": "function_call" - }, - { - "call_id": "call_6zEBnJRmXFIkGFwNjWLZP4uC", - "output": "total 4\ndrwxr-xr-x 3 vercel-sandbox vercel-sandbox 20 Aug 3 09:10 .\ndrwxr-xr-x 10 vercel-sandbox vercel-sandbox 4096 Aug 3 09:11 ..\ndrwxr-xr-x 3 vercel-sandbox vercel-sandbox 28 Aug 3 09:10 skills", - "type": "function_call_output" - }, - { - "arguments": "{\"command\":\"find . -maxdepth 4 -type f -name AGENTS.md -print -exec sed -n '1,200p' {} \\\\;\",\"timeout\":100000}", - "call_id": "call_jQUCie0ZgaXUrZCDcezp0msl", - "id": "fc_054a16a2f2ea3217016a705b67575881a3aacde02b9053a321", - "name": "bash", - "type": "function_call" - }, - { - "call_id": "call_jQUCie0ZgaXUrZCDcezp0msl", - "output": "(no output)", - "type": "function_call_output" - }, - { - "arguments": "{\"command\":\"find . -maxdepth 6 -type f -name 'AGENTS.md' -print\",\"timeout\":100000}", - "call_id": "call_NgMSUb6wIR0GAf9U8LXERtBQ", - "id": "fc_054a16a2f2ea3217016a705b68c39881a3b1e518599236e41a", - "name": "bash", - "type": "function_call" - }, - { - "call_id": "call_NgMSUb6wIR0GAf9U8LXERtBQ", - "output": "(no output)", - "type": "function_call_output" - }, - { - "arguments": "{\"command\":\"find . -maxdepth 5 -type f -name '*flue*skill*' -o -name '*skill*' | head\",\"timeout\":100000}", - "call_id": "call_BXKiQ5KG8c42XPPp94zgcY3H", - "id": "fc_054a16a2f2ea3217016a705b69edd081a38c31583ec56eca5f", - "name": "bash", - "type": "function_call" - }, - { - "call_id": "call_BXKiQ5KG8c42XPPp94zgcY3H", - "output": "./.agents/skills\n./.agents/skills/e2e-flue-skill", - "type": "function_call_output" - }, - { - "arguments": "{\"command\":\"find ./.agents/skills/e2e-flue-skill -maxdepth 2 -type f -print\",\"timeout\":100000}", - "call_id": "call_Zmmrs64LFscpAHsIUaIfEcAN", - "id": "fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63", - "name": "bash", - "type": "function_call" - }, - { - "call_id": "call_Zmmrs64LFscpAHsIUaIfEcAN", - "output": "total 4\ndrwxr-xr-x 2 vercel-sandbox vercel-sandbox 22 Aug 3 07:55 .\ndrwxr-xr-x 3 vercel-sandbox vercel-sandbox 28 Aug 3 07:55 ..\n-rw-r--r-- 1 vercel-sandbox vercel-sandbox 194 Aug 3 07:55 SKILL.md\n./.agents/skills/e2e-flue-skill/SKILL.md", - "type": "function_call_output" - }, - { - "arguments": "{\"path\":\"./.agents/skills/e2e-flue-skill/SKILL.md\",\"offset\":1,\"limit\":200}", - "call_id": "call_KXcpmIy4Cbfugk4SNT2Ge7sG", - "id": "fc_054a16a2f2ea3217016a705b6d71cc81a38ac06634a309a997", - "name": "read", - "type": "function_call" - }, - { - "call_id": "call_KXcpmIy4Cbfugk4SNT2Ge7sG", - "output": "---\nname: e2e-flue-skill\ndescription: Deterministic skill used by the Braintrust Flue instrumentation e2e test.\n---\n\nReturn the marker from the provided args exactly once. Output no other text.\n", - "type": "function_call_output" - } - ], - "model": "gpt-5.4-nano", - "prompt_cache_key": "[REDACTED]", - "reasoning": { - "effort": "none" - }, - "store": false, - "stream": true, - "tools": [ - { - "description": "Read a file or list a directory. For files, output is truncated to 2000 lines or 50KB — use offset/limit for large files. For directories, returns the list of entries.", - "name": "read", - "parameters": { - "properties": { - "limit": { - "description": "Maximum number of lines to read", - "type": "number" - }, - "offset": { - "description": "Line number to start from (1-indexed)", - "type": "number" - }, - "path": { - "description": "Path to the file to read", - "type": "string" - } - }, - "required": ["path"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Write content to a file. Creates the file and parent directories if they do not exist.", - "name": "write", - "parameters": { - "properties": { - "content": { - "description": "Content to write to the file", - "type": "string" - }, - "path": { - "description": "Path to the file to write", - "type": "string" - } - }, - "required": ["path", "content"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Edit a file using exact text replacement. The oldText must match a unique region of the file. Use replaceAll to replace all occurrences.", - "name": "edit", - "parameters": { - "properties": { - "newText": { - "description": "Replacement text", - "type": "string" - }, - "oldText": { - "description": "Exact text to find (must be unique)", - "type": "string" - }, - "path": { - "description": "Path to the file to edit", - "type": "string" - }, - "replaceAll": { - "description": "Replace all occurrences", - "type": "boolean" - } - }, - "required": ["path", "oldText", "newText"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Execute a bash command. Returns stdout and stderr. Output is truncated to the last 2000 lines or 50KB.", - "name": "bash", - "parameters": { - "properties": { - "command": { - "description": "Bash command to execute", - "type": "string" - }, - "timeout": { - "description": "Timeout in seconds", - "type": "number" - } - }, - "required": ["command"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Search file contents for a regex pattern. Returns matching lines with file paths and line numbers.", - "name": "grep", - "parameters": { - "properties": { - "include": { - "description": "Glob filter, e.g. \"*.ts\"", - "type": "string" - }, - "path": { - "description": "Directory or file to search (default: .)", - "type": "string" - }, - "pattern": { - "description": "Search pattern (regex)", - "type": "string" - } - }, - "required": ["pattern"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Find files by filename pattern using shell find -name semantics. Returns matching file paths.", - "name": "glob", - "parameters": { - "properties": { - "path": { - "description": "Directory to search in (default: .)", - "type": "string" - }, - "pattern": { - "description": "Filename pattern, e.g. \"*.ts\"", - "type": "string" - } - }, - "required": ["pattern"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Delegate a focused task to a detached child agent with its own context. Use this for independent research, file exploration, or parallel work. The task returns only its final answer to this conversation. No subagents are currently defined.", - "name": "task", - "parameters": { - "properties": { - "agent": { - "description": "Declared subagent to use for the child agent", - "type": "string" - }, - "cwd": { - "description": "Working directory for the child agent. AGENTS.md and skills are discovered from here.", - "type": "string" - }, - "description": { - "description": "Short human-readable label for the delegated work", - "type": "string" - }, - "prompt": { - "description": "Focused instructions for the child agent", - "type": "string" - } - }, - "required": ["prompt"], - "type": "object" - }, - "strict": false, - "type": "function" - } - ] - } - }, - "headers": {}, - "method": "POST", - "url": "https://api.openai.com/v1/responses" - }, - "response": { - "body": { - "chunks": [ - "event: response.created\ndata: {\"type\":\"response.created\",\"response\":{\"id\":\"resp_054a16a2f2ea3217016a705b6e34cc81a383184de2af428d09\",\"object\":\"response\",\"created_at\":1785748334,\"status\":\"in_progress\",\"background\":false,\"completed_at\":null,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":null,\"max_tool_calls\":null,\"model\":\"gpt-5.4-nano-2026-03-17\",\"moderation\":null,\"output\":[],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":\"workflow:instrumentation:01KZ3E9TWTAEK4P4P3F63HKAKC::default::ta\",\"prompt_cache_retention\":\"24h\",\"reasoning\":{\"context\":\"current_turn\",\"effort\":\"none\",\"mode\":\"standard\",\"summary\":null},\"safety_identifier\":null,\"service_tier\":\"auto\",\"store\":false,\"temperature\":1,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tool_usage\":{\"image_gen\":{\"input_tokens\":0,\"input_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"output_tokens\":0,\"output_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"total_tokens\":0},\"web_search\":{\"num_requests\":0}},\"tools\":[{\"type\":\"function\",\"description\":\"Read a file or list a directory. For files, output is truncated to 2000 lines or 50KB — use offset/limit for large files. For directories, returns the list of entries.\",\"name\":\"read\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to read\"},\"offset\":{\"type\":\"number\",\"description\":\"Line number to start from (1-indexed)\"},\"limit\":{\"type\":\"number\",\"description\":\"Maximum number of lines to read\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Write content to a file. Creates the file and parent directories if they do not exist.\",\"name\":\"write\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"content\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to write\"},\"content\":{\"type\":\"string\",\"description\":\"Content to write to the file\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Edit a file using exact text replacement. The oldText must match a unique region of the file. Use replaceAll to replace all occurrences.\",\"name\":\"edit\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"oldText\",\"newText\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to edit\"},\"oldText\":{\"type\":\"string\",\"description\":\"Exact text to find (must be unique)\"},\"newText\":{\"type\":\"string\",\"description\":\"Replacement text\"},\"replaceAll\":{\"type\":\"boolean\",\"description\":\"Replace all occurrences\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Execute a bash command. Returns stdout and stderr. Output is truncated to the last 2000 lines or 50KB.\",\"name\":\"bash\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"command\"],\"properties\":{\"command\":{\"type\":\"string\",\"description\":\"Bash command to execute\"},\"timeout\":{\"type\":\"number\",\"description\":\"Timeout in seconds\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Search file contents for a regex pattern. Returns matching lines with file paths and line numbers.\",\"name\":\"grep\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Search pattern (regex)\"},\"path\":{\"type\":\"string\",\"description\":\"Directory or file to search (default: .)\"},\"include\":{\"type\":\"string\",\"description\":\"Glob filter, e.g. \\\"*.ts\\\"\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Find files by filename pattern using shell find -name semantics. Returns matching file paths.\",\"name\":\"glob\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Filename pattern, e.g. \\\"*.ts\\\"\"},\"path\":{\"type\":\"string\",\"description\":\"Directory to search in (default: .)\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Delegate a focused task to a detached child agent with its own context. Use this for independent research, file exploration, or parallel work. The task returns only its final answer to this conversation. No subagents are currently defined.\",\"name\":\"task\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"prompt\"],\"properties\":{\"description\":{\"type\":\"string\",\"description\":\"Short human-readable label for the delegated work\"},\"prompt\":{\"type\":\"string\",\"description\":\"Focused instructions for the child agent\"},\"agent\":{\"type\":\"string\",\"description\":\"Declared subagent to use for the child agent\"},\"cwd\":{\"type\":\"string\",\"description\":\"Working directory for the child agent. AGENTS.md and skills are discovered from here.\"}}},\"strict\":false}],\"top_logprobs\":0,\"top_p\":0.98,\"truncation\":\"disabled\",\"usage\":null,\"user\":null,\"metadata\":{}},\"sequence_number\":0}", - "event: response.in_progress\ndata: {\"type\":\"response.in_progress\",\"response\":{\"id\":\"resp_054a16a2f2ea3217016a705b6e34cc81a383184de2af428d09\",\"object\":\"response\",\"created_at\":1785748334,\"status\":\"in_progress\",\"background\":false,\"completed_at\":null,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":null,\"max_tool_calls\":null,\"model\":\"gpt-5.4-nano-2026-03-17\",\"moderation\":null,\"output\":[],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":\"workflow:instrumentation:01KZ3E9TWTAEK4P4P3F63HKAKC::default::ta\",\"prompt_cache_retention\":\"24h\",\"reasoning\":{\"context\":\"current_turn\",\"effort\":\"none\",\"mode\":\"standard\",\"summary\":null},\"safety_identifier\":null,\"service_tier\":\"auto\",\"store\":false,\"temperature\":1,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tool_usage\":{\"image_gen\":{\"input_tokens\":0,\"input_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"output_tokens\":0,\"output_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"total_tokens\":0},\"web_search\":{\"num_requests\":0}},\"tools\":[{\"type\":\"function\",\"description\":\"Read a file or list a directory. For files, output is truncated to 2000 lines or 50KB — use offset/limit for large files. For directories, returns the list of entries.\",\"name\":\"read\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to read\"},\"offset\":{\"type\":\"number\",\"description\":\"Line number to start from (1-indexed)\"},\"limit\":{\"type\":\"number\",\"description\":\"Maximum number of lines to read\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Write content to a file. Creates the file and parent directories if they do not exist.\",\"name\":\"write\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"content\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to write\"},\"content\":{\"type\":\"string\",\"description\":\"Content to write to the file\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Edit a file using exact text replacement. The oldText must match a unique region of the file. Use replaceAll to replace all occurrences.\",\"name\":\"edit\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"oldText\",\"newText\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to edit\"},\"oldText\":{\"type\":\"string\",\"description\":\"Exact text to find (must be unique)\"},\"newText\":{\"type\":\"string\",\"description\":\"Replacement text\"},\"replaceAll\":{\"type\":\"boolean\",\"description\":\"Replace all occurrences\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Execute a bash command. Returns stdout and stderr. Output is truncated to the last 2000 lines or 50KB.\",\"name\":\"bash\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"command\"],\"properties\":{\"command\":{\"type\":\"string\",\"description\":\"Bash command to execute\"},\"timeout\":{\"type\":\"number\",\"description\":\"Timeout in seconds\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Search file contents for a regex pattern. Returns matching lines with file paths and line numbers.\",\"name\":\"grep\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Search pattern (regex)\"},\"path\":{\"type\":\"string\",\"description\":\"Directory or file to search (default: .)\"},\"include\":{\"type\":\"string\",\"description\":\"Glob filter, e.g. \\\"*.ts\\\"\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Find files by filename pattern using shell find -name semantics. Returns matching file paths.\",\"name\":\"glob\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Filename pattern, e.g. \\\"*.ts\\\"\"},\"path\":{\"type\":\"string\",\"description\":\"Directory to search in (default: .)\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Delegate a focused task to a detached child agent with its own context. Use this for independent research, file exploration, or parallel work. The task returns only its final answer to this conversation. No subagents are currently defined.\",\"name\":\"task\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"prompt\"],\"properties\":{\"description\":{\"type\":\"string\",\"description\":\"Short human-readable label for the delegated work\"},\"prompt\":{\"type\":\"string\",\"description\":\"Focused instructions for the child agent\"},\"agent\":{\"type\":\"string\",\"description\":\"Declared subagent to use for the child agent\"},\"cwd\":{\"type\":\"string\",\"description\":\"Working directory for the child agent. AGENTS.md and skills are discovered from here.\"}}},\"strict\":false}],\"top_logprobs\":0,\"top_p\":0.98,\"truncation\":\"disabled\",\"usage\":null,\"user\":null,\"metadata\":{}},\"sequence_number\":1}", - "event: response.output_item.added\ndata: {\"type\":\"response.output_item.added\",\"item\":{\"id\":\"fc_054a16a2f2ea3217016a705b6e90f481a39bc44496fbadf441\",\"type\":\"function_call\",\"status\":\"in_progress\",\"arguments\":\"\",\"call_id\":\"call_xQybhnQYuflmfp7gyAScVKjY\",\"name\":\"bash\"},\"output_index\":0,\"sequence_number\":2}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"{\\\"\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6e90f481a39bc44496fbadf441\",\"obfuscation\":\"IsfKDqHlaKpPlc\",\"output_index\":0,\"sequence_number\":3}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"command\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6e90f481a39bc44496fbadf441\",\"obfuscation\":\"05wTcU2vE\",\"output_index\":0,\"sequence_number\":4}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"\\\":\\\"\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6e90f481a39bc44496fbadf441\",\"obfuscation\":\"I7n2P9cIUaGZ5\",\"output_index\":0,\"sequence_number\":5}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"node\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6e90f481a39bc44496fbadf441\",\"obfuscation\":\"66No8n2S3lQV\",\"output_index\":0,\"sequence_number\":6}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" -\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6e90f481a39bc44496fbadf441\",\"obfuscation\":\"pfQfFqmrd7Cttd\",\"output_index\":0,\"sequence_number\":7}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"v\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6e90f481a39bc44496fbadf441\",\"obfuscation\":\"MsjXV30TNBXaVIN\",\"output_index\":0,\"sequence_number\":8}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" &&\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6e90f481a39bc44496fbadf441\",\"obfuscation\":\"sbVmMrYzWCmWN\",\"output_index\":0,\"sequence_number\":9}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" ls\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6e90f481a39bc44496fbadf441\",\"obfuscation\":\"f3AOiA6bYx0WA\",\"output_index\":0,\"sequence_number\":10}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" -\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6e90f481a39bc44496fbadf441\",\"obfuscation\":\"YwMIVfxBwLPyPz\",\"output_index\":0,\"sequence_number\":11}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"la\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6e90f481a39bc44496fbadf441\",\"obfuscation\":\"yosVAZ9d354t0E\",\"output_index\":0,\"sequence_number\":12}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" /\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6e90f481a39bc44496fbadf441\",\"obfuscation\":\"RtzYTt0vUd1KL5\",\"output_index\":0,\"sequence_number\":13}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"ver\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6e90f481a39bc44496fbadf441\",\"obfuscation\":\"UZCqFJvIdZJfi\",\"output_index\":0,\"sequence_number\":14}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"cel\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6e90f481a39bc44496fbadf441\",\"obfuscation\":\"ob8WdL1Zrd6z4\",\"output_index\":0,\"sequence_number\":15}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/s\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6e90f481a39bc44496fbadf441\",\"obfuscation\":\"SHFL6SDvDru2gO\",\"output_index\":0,\"sequence_number\":16}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"andbox\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6e90f481a39bc44496fbadf441\",\"obfuscation\":\"SlANW27QBQ\",\"output_index\":0,\"sequence_number\":17}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6e90f481a39bc44496fbadf441\",\"obfuscation\":\"Llwwd75yCVN6cFY\",\"output_index\":0,\"sequence_number\":18}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"brain\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6e90f481a39bc44496fbadf441\",\"obfuscation\":\"Sp2PfwThtXK\",\"output_index\":0,\"sequence_number\":19}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"trust\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6e90f481a39bc44496fbadf441\",\"obfuscation\":\"WTr9EL0CMfU\",\"output_index\":0,\"sequence_number\":20}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"data\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6e90f481a39bc44496fbadf441\",\"obfuscation\":\"iXd0y36BKeMz\",\"output_index\":0,\"sequence_number\":21}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6e90f481a39bc44496fbadf441\",\"obfuscation\":\"TMVJ4avBWNU1PMg\",\"output_index\":0,\"sequence_number\":22}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"brain\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6e90f481a39bc44496fbadf441\",\"obfuscation\":\"gGYEjs7floV\",\"output_index\":0,\"sequence_number\":23}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"trust\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6e90f481a39bc44496fbadf441\",\"obfuscation\":\"nJfWpNMEeYh\",\"output_index\":0,\"sequence_number\":24}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"-sdk\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6e90f481a39bc44496fbadf441\",\"obfuscation\":\"VnnFujipwf0B\",\"output_index\":0,\"sequence_number\":25}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"-j\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6e90f481a39bc44496fbadf441\",\"obfuscation\":\"AAXM8t27HpiY8a\",\"output_index\":0,\"sequence_number\":26}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"avascript\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6e90f481a39bc44496fbadf441\",\"obfuscation\":\"51XrioX\",\"output_index\":0,\"sequence_number\":27}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/e\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6e90f481a39bc44496fbadf441\",\"obfuscation\":\"zAaUUwSEMO3Bxs\",\"output_index\":0,\"sequence_number\":28}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"2\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6e90f481a39bc44496fbadf441\",\"obfuscation\":\"vyBpS3fu1undfoT\",\"output_index\":0,\"sequence_number\":29}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"e\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6e90f481a39bc44496fbadf441\",\"obfuscation\":\"myDkZQqFtQPgErv\",\"output_index\":0,\"sequence_number\":30}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/sc\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6e90f481a39bc44496fbadf441\",\"obfuscation\":\"7jYdsKtAqGJsx\",\"output_index\":0,\"sequence_number\":31}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"enarios\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6e90f481a39bc44496fbadf441\",\"obfuscation\":\"FgfnvD0oV\",\"output_index\":0,\"sequence_number\":32}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/fl\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6e90f481a39bc44496fbadf441\",\"obfuscation\":\"ECM3KkIZxcc2j\",\"output_index\":0,\"sequence_number\":33}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"ue\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6e90f481a39bc44496fbadf441\",\"obfuscation\":\"XIzDZYr4JpAjhV\",\"output_index\":0,\"sequence_number\":34}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"-in\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6e90f481a39bc44496fbadf441\",\"obfuscation\":\"VudZPLnjNH5PJ\",\"output_index\":0,\"sequence_number\":35}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"strument\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6e90f481a39bc44496fbadf441\",\"obfuscation\":\"i2iTyJpP\",\"output_index\":0,\"sequence_number\":36}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"ation\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6e90f481a39bc44496fbadf441\",\"obfuscation\":\"ZtblFC8j0Wk\",\"output_index\":0,\"sequence_number\":37}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" &&\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6e90f481a39bc44496fbadf441\",\"obfuscation\":\"1l1tQ40NrGj8E\",\"output_index\":0,\"sequence_number\":38}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" cat\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6e90f481a39bc44496fbadf441\",\"obfuscation\":\"41U8EU6Ib9cg\",\"output_index\":0,\"sequence_number\":39}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" /\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6e90f481a39bc44496fbadf441\",\"obfuscation\":\"cglsLI76yH9ClQ\",\"output_index\":0,\"sequence_number\":40}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"ver\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6e90f481a39bc44496fbadf441\",\"obfuscation\":\"qvk54uq2LZnZa\",\"output_index\":0,\"sequence_number\":41}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"cel\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6e90f481a39bc44496fbadf441\",\"obfuscation\":\"aovMJj1uFOFEM\",\"output_index\":0,\"sequence_number\":42}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/s\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6e90f481a39bc44496fbadf441\",\"obfuscation\":\"NQpTAggULtv8zO\",\"output_index\":0,\"sequence_number\":43}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"andbox\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6e90f481a39bc44496fbadf441\",\"obfuscation\":\"5837PUGg7V\",\"output_index\":0,\"sequence_number\":44}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6e90f481a39bc44496fbadf441\",\"obfuscation\":\"JhMbsWIyEoPXZRB\",\"output_index\":0,\"sequence_number\":45}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"brain\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6e90f481a39bc44496fbadf441\",\"obfuscation\":\"wUKGqlUZBXL\",\"output_index\":0,\"sequence_number\":46}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"trust\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6e90f481a39bc44496fbadf441\",\"obfuscation\":\"5fqXYCtg0Gt\",\"output_index\":0,\"sequence_number\":47}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"data\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6e90f481a39bc44496fbadf441\",\"obfuscation\":\"4Q1cI53ucyWL\",\"output_index\":0,\"sequence_number\":48}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6e90f481a39bc44496fbadf441\",\"obfuscation\":\"abd5F6mWTBD3FqR\",\"output_index\":0,\"sequence_number\":49}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"brain\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6e90f481a39bc44496fbadf441\",\"obfuscation\":\"YwaHJecYa6Z\",\"output_index\":0,\"sequence_number\":50}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"trust\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6e90f481a39bc44496fbadf441\",\"obfuscation\":\"IZiyFBgwGNk\",\"output_index\":0,\"sequence_number\":51}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"-sdk\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6e90f481a39bc44496fbadf441\",\"obfuscation\":\"DefferqgDs1G\",\"output_index\":0,\"sequence_number\":52}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"-j\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6e90f481a39bc44496fbadf441\",\"obfuscation\":\"jf9cr3akDjVA60\",\"output_index\":0,\"sequence_number\":53}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"avascript\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6e90f481a39bc44496fbadf441\",\"obfuscation\":\"XaGvBNu\",\"output_index\":0,\"sequence_number\":54}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/e\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6e90f481a39bc44496fbadf441\",\"obfuscation\":\"5UiGHyP1PGgpsP\",\"output_index\":0,\"sequence_number\":55}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"2\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6e90f481a39bc44496fbadf441\",\"obfuscation\":\"WKaJqwMQWdtBVYa\",\"output_index\":0,\"sequence_number\":56}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"e\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6e90f481a39bc44496fbadf441\",\"obfuscation\":\"DiOuTdnwYCCUpLq\",\"output_index\":0,\"sequence_number\":57}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/sc\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6e90f481a39bc44496fbadf441\",\"obfuscation\":\"JNIox4EYrQqid\",\"output_index\":0,\"sequence_number\":58}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"enarios\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6e90f481a39bc44496fbadf441\",\"obfuscation\":\"rQexF5fiq\",\"output_index\":0,\"sequence_number\":59}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/fl\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6e90f481a39bc44496fbadf441\",\"obfuscation\":\"CxjB3QQkwwxPq\",\"output_index\":0,\"sequence_number\":60}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"ue\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6e90f481a39bc44496fbadf441\",\"obfuscation\":\"KvqTdaudMieVrZ\",\"output_index\":0,\"sequence_number\":61}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"-in\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6e90f481a39bc44496fbadf441\",\"obfuscation\":\"hqrEuju9Hubvj\",\"output_index\":0,\"sequence_number\":62}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"strument\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6e90f481a39bc44496fbadf441\",\"obfuscation\":\"xqdV3tk5\",\"output_index\":0,\"sequence_number\":63}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"ation\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6e90f481a39bc44496fbadf441\",\"obfuscation\":\"finNWbOcOSK\",\"output_index\":0,\"sequence_number\":64}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/package\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6e90f481a39bc44496fbadf441\",\"obfuscation\":\"w5O538dR\",\"output_index\":0,\"sequence_number\":65}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\".json\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6e90f481a39bc44496fbadf441\",\"obfuscation\":\"5lixmLK9osS\",\"output_index\":0,\"sequence_number\":66}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" \",\"item_id\":\"fc_054a16a2f2ea3217016a705b6e90f481a39bc44496fbadf441\",\"obfuscation\":\"UfcwAy15pK3UyDX\",\"output_index\":0,\"sequence_number\":67}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"2\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6e90f481a39bc44496fbadf441\",\"obfuscation\":\"8kuEminD3O9nEe6\",\"output_index\":0,\"sequence_number\":68}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\">/\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6e90f481a39bc44496fbadf441\",\"obfuscation\":\"0ujGWvOIOaHg9t\",\"output_index\":0,\"sequence_number\":69}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"dev\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6e90f481a39bc44496fbadf441\",\"obfuscation\":\"l87mZp3YVC56A\",\"output_index\":0,\"sequence_number\":70}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/null\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6e90f481a39bc44496fbadf441\",\"obfuscation\":\"z1OHwKMfyTN\",\"output_index\":0,\"sequence_number\":71}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" ||\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6e90f481a39bc44496fbadf441\",\"obfuscation\":\"Nd0PVbdoXgVdP\",\"output_index\":0,\"sequence_number\":72}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" true\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6e90f481a39bc44496fbadf441\",\"obfuscation\":\"gYO8swE9D00\",\"output_index\":0,\"sequence_number\":73}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"\\\",\\\"\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6e90f481a39bc44496fbadf441\",\"obfuscation\":\"dAlT06VEGUSd4\",\"output_index\":0,\"sequence_number\":74}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"timeout\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6e90f481a39bc44496fbadf441\",\"obfuscation\":\"9zDDLkKve\",\"output_index\":0,\"sequence_number\":75}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"\\\":\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6e90f481a39bc44496fbadf441\",\"obfuscation\":\"bOgN2cbK9QPYUV\",\"output_index\":0,\"sequence_number\":76}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"100\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6e90f481a39bc44496fbadf441\",\"obfuscation\":\"2ycAaEUguB849\",\"output_index\":0,\"sequence_number\":77}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"000\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6e90f481a39bc44496fbadf441\",\"obfuscation\":\"jCkJ68Rcb58dt\",\"output_index\":0,\"sequence_number\":78}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"}\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6e90f481a39bc44496fbadf441\",\"obfuscation\":\"bbF8wOnG3Zvtgvs\",\"output_index\":0,\"sequence_number\":79}", - "event: response.function_call_arguments.done\ndata: {\"type\":\"response.function_call_arguments.done\",\"arguments\":\"{\\\"command\\\":\\\"cat ./package.json\\\",\\\"timeout\\\":100000}\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6e90f481a39bc44496fbadf441\",\"output_index\":0,\"sequence_number\":80}", - "event: response.output_item.done\ndata: {\"type\":\"response.output_item.done\",\"item\":{\"id\":\"fc_054a16a2f2ea3217016a705b6e90f481a39bc44496fbadf441\",\"type\":\"function_call\",\"status\":\"completed\",\"arguments\":\"{\\\"command\\\":\\\"cat ./package.json\\\",\\\"timeout\\\":100000}\",\"call_id\":\"call_xQybhnQYuflmfp7gyAScVKjY\",\"name\":\"bash\"},\"output_index\":0,\"sequence_number\":81}", - "event: response.completed\ndata: {\"type\":\"response.completed\",\"response\":{\"id\":\"resp_054a16a2f2ea3217016a705b6e34cc81a383184de2af428d09\",\"object\":\"response\",\"created_at\":1785748334,\"status\":\"completed\",\"background\":false,\"completed_at\":1785748335,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":null,\"max_tool_calls\":null,\"model\":\"gpt-5.4-nano-2026-03-17\",\"moderation\":null,\"output\":[{\"id\":\"fc_054a16a2f2ea3217016a705b6e90f481a39bc44496fbadf441\",\"type\":\"function_call\",\"status\":\"completed\",\"arguments\":\"{\\\"command\\\":\\\"cat ./package.json\\\",\\\"timeout\\\":100000}\",\"call_id\":\"call_xQybhnQYuflmfp7gyAScVKjY\",\"name\":\"bash\"}],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":\"workflow:instrumentation:01KZ3E9TWTAEK4P4P3F63HKAKC::default::ta\",\"prompt_cache_retention\":\"24h\",\"reasoning\":{\"context\":\"current_turn\",\"effort\":\"none\",\"mode\":\"standard\",\"summary\":null},\"safety_identifier\":null,\"service_tier\":\"default\",\"store\":false,\"temperature\":1,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tool_usage\":{\"image_gen\":{\"input_tokens\":0,\"input_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"output_tokens\":0,\"output_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"total_tokens\":0},\"web_search\":{\"num_requests\":0}},\"tools\":[{\"type\":\"function\",\"description\":\"Read a file or list a directory. For files, output is truncated to 2000 lines or 50KB — use offset/limit for large files. For directories, returns the list of entries.\",\"name\":\"read\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to read\"},\"offset\":{\"type\":\"number\",\"description\":\"Line number to start from (1-indexed)\"},\"limit\":{\"type\":\"number\",\"description\":\"Maximum number of lines to read\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Write content to a file. Creates the file and parent directories if they do not exist.\",\"name\":\"write\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"content\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to write\"},\"content\":{\"type\":\"string\",\"description\":\"Content to write to the file\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Edit a file using exact text replacement. The oldText must match a unique region of the file. Use replaceAll to replace all occurrences.\",\"name\":\"edit\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"oldText\",\"newText\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to edit\"},\"oldText\":{\"type\":\"string\",\"description\":\"Exact text to find (must be unique)\"},\"newText\":{\"type\":\"string\",\"description\":\"Replacement text\"},\"replaceAll\":{\"type\":\"boolean\",\"description\":\"Replace all occurrences\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Execute a bash command. Returns stdout and stderr. Output is truncated to the last 2000 lines or 50KB.\",\"name\":\"bash\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"command\"],\"properties\":{\"command\":{\"type\":\"string\",\"description\":\"Bash command to execute\"},\"timeout\":{\"type\":\"number\",\"description\":\"Timeout in seconds\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Search file contents for a regex pattern. Returns matching lines with file paths and line numbers.\",\"name\":\"grep\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Search pattern (regex)\"},\"path\":{\"type\":\"string\",\"description\":\"Directory or file to search (default: .)\"},\"include\":{\"type\":\"string\",\"description\":\"Glob filter, e.g. \\\"*.ts\\\"\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Find files by filename pattern using shell find -name semantics. Returns matching file paths.\",\"name\":\"glob\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Filename pattern, e.g. \\\"*.ts\\\"\"},\"path\":{\"type\":\"string\",\"description\":\"Directory to search in (default: .)\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Delegate a focused task to a detached child agent with its own context. Use this for independent research, file exploration, or parallel work. The task returns only its final answer to this conversation. No subagents are currently defined.\",\"name\":\"task\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"prompt\"],\"properties\":{\"description\":{\"type\":\"string\",\"description\":\"Short human-readable label for the delegated work\"},\"prompt\":{\"type\":\"string\",\"description\":\"Focused instructions for the child agent\"},\"agent\":{\"type\":\"string\",\"description\":\"Declared subagent to use for the child agent\"},\"cwd\":{\"type\":\"string\",\"description\":\"Working directory for the child agent. AGENTS.md and skills are discovered from here.\"}}},\"strict\":false}],\"top_logprobs\":0,\"top_p\":0.98,\"truncation\":\"disabled\",\"usage\":{\"input_tokens\":1882,\"input_tokens_details\":{\"cache_write_tokens\":0,\"cached_tokens\":1536},\"output_tokens\":90,\"output_tokens_details\":{\"reasoning_tokens\":0},\"total_tokens\":1972},\"user\":null,\"metadata\":{}},\"sequence_number\":82}" - ], - "kind": "sse" - }, - "headers": { - "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", - "alt-svc": "h3=\":443\"; ma=86400", - "cf-cache-status": "DYNAMIC", - "cf-ray": "a25433108e2d7bb6-IAD", - "connection": "keep-alive", - "content-type": "text/event-stream; charset=utf-8", - "date": "Mon, 03 Aug 2026 09:12:14 GMT", - "openai-organization": "braintrust-data", - "openai-processing-ms": "170", - "openai-project": "proj_vsCSXafhhByzWOThMrJcZiw9", - "openai-version": "2020-10-01", - "server": "cloudflare", - "set-cookie": "[REDACTED]", - "strict-transport-security": "max-age=31536000; includeSubDomains; preload", - "transfer-encoding": "chunked", - "x-content-type-options": "nosniff", - "x-ratelimit-limit-requests": "30000", - "x-ratelimit-limit-tokens": "180000000", - "x-ratelimit-remaining-requests": "29999", - "x-ratelimit-remaining-tokens": "179997663", - "x-ratelimit-reset-requests": "2ms", - "x-ratelimit-reset-tokens": "0s", - "x-request-id": "req_be4542a23d5f4e17b4285d598ac2400b" - }, - "status": 200, - "statusText": "OK" - } - }, - { - "callIndex": 13, - "id": "eebc152ba05b8412", - "matchKey": "POST api.openai.com/v1/responses", - "recordedAt": "2026-08-03T09:12:16.175Z", - "request": { - "body": { - "kind": "json", - "value": { - "input": [ - { - "content": "You are running in headless mode with no human operator. Work autonomously — never ask questions, never wait for user input. Make your best judgment and proceed independently.\n\nYou are a deterministic Flue instrumentation test agent. Follow user instructions exactly. When asked for a marker, output only that marker and no extra text. When running a local skill file, read it yourself and do not delegate it to a task.\n\n## Available Skills\n\nThe following skills provide specialized instructions for specific tasks. When a task matches a skill description, activate that skill before proceeding so its full instructions are loaded. Skill instructions and supporting resources stay lazy until activation or explicit file reads.\n\n- **e2e-flue-skill** — Deterministic skill used by the Braintrust Flue instrumentation e2e test.\n\nDate: Mon, Aug 3, 2026\nWorking directory: .\n\nDirectory structure:\n.agents\n.flue\n.flue-build\n__cassettes__\n__snapshots__\nassertions.ts\ncassette-filter.mjs\nconstants.mjs\ndist\nnode_modules\npackage.json\npnpm-lock.yaml\nscenario.cli.mjs\nscenario.impl.mjs\nscenario.mjs\nscenario.test.ts\nscenario.ts\nscenario.v2.ts\nv2-assertions.ts\nversions", - "role": "developer" - }, - { - "content": [ - { - "text": "Run the skill named \"e2e-flue-skill\" with arguments {\"marker\":\"SKILL_DONE\"}. Output only the marker and no extra text when finished.", - "type": "input_text" - } - ], - "role": "user" - }, - { - "arguments": "{\"description\":\"Run e2e-flue-skill with marker SKILL_DONE\",\"prompt\":\"Run the skill named \\\"e2e-flue-skill\\\" with arguments {\\\"marker\\\":\\\"SKILL_DONE\\\"}. Ensure completion and report the final output marker only.\",\"agent\":\"e2e-flue-skill\",\"cwd\":\".\"}", - "call_id": "call_w6pyYZgnhlSOAXul75DAG1Cj", - "id": "fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6", - "name": "task", - "type": "function_call" - }, - { - "call_id": "call_w6pyYZgnhlSOAXul75DAG1Cj", - "output": "[flue] Subagent \"e2e-flue-skill\" is not declared. Available: (none).", - "type": "function_call_output" - }, - { - "arguments": "{\"command\":\"find . -maxdepth 4 -name AGENTS.md -print\",\"timeout\":100000}", - "call_id": "call_6zEBnJRmXFIkGFwNjWLZP4uC", - "id": "fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee", - "name": "bash", - "type": "function_call" - }, - { - "call_id": "call_6zEBnJRmXFIkGFwNjWLZP4uC", - "output": "total 4\ndrwxr-xr-x 3 vercel-sandbox vercel-sandbox 20 Aug 3 09:10 .\ndrwxr-xr-x 10 vercel-sandbox vercel-sandbox 4096 Aug 3 09:11 ..\ndrwxr-xr-x 3 vercel-sandbox vercel-sandbox 28 Aug 3 09:10 skills", - "type": "function_call_output" - }, - { - "arguments": "{\"command\":\"find . -maxdepth 4 -type f -name AGENTS.md -print -exec sed -n '1,200p' {} \\\\;\",\"timeout\":100000}", - "call_id": "call_jQUCie0ZgaXUrZCDcezp0msl", - "id": "fc_054a16a2f2ea3217016a705b67575881a3aacde02b9053a321", - "name": "bash", - "type": "function_call" - }, - { - "call_id": "call_jQUCie0ZgaXUrZCDcezp0msl", - "output": "(no output)", - "type": "function_call_output" - }, - { - "arguments": "{\"command\":\"find . -maxdepth 6 -type f -name 'AGENTS.md' -print\",\"timeout\":100000}", - "call_id": "call_NgMSUb6wIR0GAf9U8LXERtBQ", - "id": "fc_054a16a2f2ea3217016a705b68c39881a3b1e518599236e41a", - "name": "bash", - "type": "function_call" - }, - { - "call_id": "call_NgMSUb6wIR0GAf9U8LXERtBQ", - "output": "(no output)", - "type": "function_call_output" - }, - { - "arguments": "{\"command\":\"find . -maxdepth 5 -type f -name '*flue*skill*' -o -name '*skill*' | head\",\"timeout\":100000}", - "call_id": "call_BXKiQ5KG8c42XPPp94zgcY3H", - "id": "fc_054a16a2f2ea3217016a705b69edd081a38c31583ec56eca5f", - "name": "bash", - "type": "function_call" - }, - { - "call_id": "call_BXKiQ5KG8c42XPPp94zgcY3H", - "output": "./.agents/skills\n./.agents/skills/e2e-flue-skill", - "type": "function_call_output" - }, - { - "arguments": "{\"command\":\"find ./.agents/skills/e2e-flue-skill -maxdepth 2 -type f -print\",\"timeout\":100000}", - "call_id": "call_Zmmrs64LFscpAHsIUaIfEcAN", - "id": "fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63", - "name": "bash", - "type": "function_call" - }, - { - "call_id": "call_Zmmrs64LFscpAHsIUaIfEcAN", - "output": "total 4\ndrwxr-xr-x 2 vercel-sandbox vercel-sandbox 22 Aug 3 07:55 .\ndrwxr-xr-x 3 vercel-sandbox vercel-sandbox 28 Aug 3 07:55 ..\n-rw-r--r-- 1 vercel-sandbox vercel-sandbox 194 Aug 3 07:55 SKILL.md\n./.agents/skills/e2e-flue-skill/SKILL.md", - "type": "function_call_output" - }, - { - "arguments": "{\"path\":\"./.agents/skills/e2e-flue-skill/SKILL.md\",\"offset\":1,\"limit\":200}", - "call_id": "call_KXcpmIy4Cbfugk4SNT2Ge7sG", - "id": "fc_054a16a2f2ea3217016a705b6d71cc81a38ac06634a309a997", - "name": "read", - "type": "function_call" - }, - { - "call_id": "call_KXcpmIy4Cbfugk4SNT2Ge7sG", - "output": "---\nname: e2e-flue-skill\ndescription: Deterministic skill used by the Braintrust Flue instrumentation e2e test.\n---\n\nReturn the marker from the provided args exactly once. Output no other text.\n", - "type": "function_call_output" - }, - { - "arguments": "{\"command\":\"cat ./package.json\",\"timeout\":100000}", - "call_id": "call_xQybhnQYuflmfp7gyAScVKjY", - "id": "fc_054a16a2f2ea3217016a705b6e90f481a39bc44496fbadf441", - "name": "bash", - "type": "function_call" - }, - { - "call_id": "call_xQybhnQYuflmfp7gyAScVKjY", - "output": "v24.14.1\ntotal 244\ndrwxr-xr-x 7 vercel-sandbox vercel-sandbox 4096 Aug 3 09:06 .\ndrwxr-xr-x 46 vercel-sandbox vercel-sandbox 4096 Aug 3 07:55 ..\ndrwxr-xr-x 3 vercel-sandbox vercel-sandbox 20 Aug 3 07:55 .agents\ndrwxr-xr-x 3 vercel-sandbox vercel-sandbox 38 Aug 3 07:55 .flue\ndrwxr-xr-x 2 vercel-sandbox vercel-sandbox 218 Aug 3 09:11 __cassettes__\ndrwxr-xr-x 2 vercel-sandbox vercel-sandbox 4096 Aug 3 08:49 __snapshots__\n-rw-r--r-- 1 vercel-sandbox vercel-sandbox 12484 Aug 3 08:45 assertions.ts\n-rw-r--r-- 1 vercel-sandbox vercel-sandbox 380 Aug 3 07:55 cassette-filter.mjs\n-rw-r--r-- 1 vercel-sandbox vercel-sandbox 215 Aug 3 07:55 constants.mjs\n-rw-r--r-- 1 vercel-sandbox vercel-sandbox 2000 Aug 3 09:06 package.json\n-rw-r--r-- 1 vercel-sandbox vercel-sandbox 167098 Aug 3 09:06 pnpm-lock.yaml\n-rw-r--r-- 1 vercel-sandbox vercel-sandbox 123 Aug 3 07:55 scenario.cli.mjs\n-rw-r--r-- 1 vercel-sandbox vercel-sandbox 6877 Aug 3 09:06 scenario.impl.mjs\n-rw-r--r-- 1 vercel-sandbox vercel-sandbox 203 Aug 3 07:55 scenario.mjs\n-rw-r--r-- 1 vercel-sandbox vercel-sandbox 8066 Aug 3 09:09 scenario.test.ts\n-rw-r--r-- 1 vercel-sandbox vercel-sandbox 121 Aug 3 07:55 scenario.ts\n-rw-r--r-- 1 vercel-sandbox vercel-sandbox 2376 Aug 3 08:49 scenario.v2.ts\n-rw-r--r-- 1 vercel-sandbox vercel-sandbox 4616 Aug 3 08:50 v2-assertions.ts\ndrwxr-xr-x 4 vercel-sandbox vercel-sandbox 26 Aug 3 08:06 versions\n{\n \"name\": \"flue-instrumentation-scenario\",\n \"private\": true,\n \"type\": \"module\",\n \"braintrustScenario\": {\n \"bump\": {\n \"dependencies\": {\n \"flue-cli-v0-8-latest\": {\n \"package\": \"@flue/cli\",\n \"range\": \">=0.8.0 <0.9.0\"\n },\n \"flue-cli-v1-latest\": {\n \"package\": \"@flue/cli\",\n \"range\": \">=1.0.0-beta.3 <2.0.0-0\",\n \"allowPrerelease\": true\n },\n \"flue-runtime-v0-8-latest\": {\n \"package\": \"@flue/runtime\",\n \"range\": \">=0.8.0 <0.9.0\"\n },\n \"flue-runtime-v1-latest\": {\n \"package\": \"@flue/runtime\",\n \"range\": \">=1.0.0-beta.3 <2.0.0-0\",\n \"allowPrerelease\": true\n },\n \"flue-runtime-v2-latest\": {\n \"package\": \"@flue/runtime\",\n \"range\": \">=2.0.0 <3.0.0\"\n }\n }\n }\n },\n \"dependencies\": {\n \"@flue/cli\": \"0.8.0\",\n \"@flue/runtime\": \"0.8.0\",\n \"flue-cli-v0-8-latest\": \"npm:@flue/cli@0.8.1\",\n \"flue-cli-v1\": \"npm:@flue/cli@1.0.0-beta.3\",\n \"flue-cli-v1-latest\": \"npm:@flue/cli@1.0.0-beta.9\",\n \"flue-runtime-v0-8-latest\": \"npm:@flue/runtime@0.8.1\",\n \"flue-runtime-v1\": \"npm:@flue/runtime@1.0.0-beta.3\",\n \"flue-runtime-v1-latest\": \"npm:@flue/runtime@1.0.0-beta.9\",\n \"flue-runtime-v2\": \"npm:@flue/runtime@2.0.0\",\n \"flue-runtime-v2-latest\": \"npm:@flue/runtime@2.0.1\",\n \"pi-ai-v2\": \"npm:@earendil-works/pi-ai@0.83.0\",\n \"valibot\": \"1.4.2\"\n },\n \"pnpm\": {\n \"overrides\": {\n \"@flue/runtime@0.8.0>@earendil-works/pi-agent-core\": \"0.75.4\",\n \"@flue/runtime@0.8.0>@earendil-works/pi-ai\": \"0.75.4\",\n \"@flue/runtime@0.8.1>@earendil-works/pi-agent-core\": \"0.75.4\",\n \"@flue/runtime@0.8.1>@earendil-works/pi-ai\": \"0.75.4\",\n \"@hono/node-server@>=2.0.0 <2.0.10\": \"^2.0.10\",\n \"fast-uri@<3.1.4\": \"^3.1.4\",\n \"hono@>=4.0.0 <4.12.27\": \"^4.12.27\",\n \"brace-expansion@>=3.0.0 <5.0.8\": \"^5.0.8\",\n \"postcss@<8.5.18\": \"^8.5.18\",\n \"valibot@<1.4.2\": \"^1.4.2\"\n }\n }\n}", - "type": "function_call_output" - } - ], - "model": "gpt-5.4-nano", - "prompt_cache_key": "[REDACTED]", - "reasoning": { - "effort": "none" - }, - "store": false, - "stream": true, - "tools": [ - { - "description": "Read a file or list a directory. For files, output is truncated to 2000 lines or 50KB — use offset/limit for large files. For directories, returns the list of entries.", - "name": "read", - "parameters": { - "properties": { - "limit": { - "description": "Maximum number of lines to read", - "type": "number" - }, - "offset": { - "description": "Line number to start from (1-indexed)", - "type": "number" - }, - "path": { - "description": "Path to the file to read", - "type": "string" - } - }, - "required": ["path"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Write content to a file. Creates the file and parent directories if they do not exist.", - "name": "write", - "parameters": { - "properties": { - "content": { - "description": "Content to write to the file", - "type": "string" - }, - "path": { - "description": "Path to the file to write", - "type": "string" - } - }, - "required": ["path", "content"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Edit a file using exact text replacement. The oldText must match a unique region of the file. Use replaceAll to replace all occurrences.", - "name": "edit", - "parameters": { - "properties": { - "newText": { - "description": "Replacement text", - "type": "string" - }, - "oldText": { - "description": "Exact text to find (must be unique)", - "type": "string" - }, - "path": { - "description": "Path to the file to edit", - "type": "string" - }, - "replaceAll": { - "description": "Replace all occurrences", - "type": "boolean" - } - }, - "required": ["path", "oldText", "newText"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Execute a bash command. Returns stdout and stderr. Output is truncated to the last 2000 lines or 50KB.", - "name": "bash", - "parameters": { - "properties": { - "command": { - "description": "Bash command to execute", - "type": "string" - }, - "timeout": { - "description": "Timeout in seconds", - "type": "number" - } - }, - "required": ["command"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Search file contents for a regex pattern. Returns matching lines with file paths and line numbers.", - "name": "grep", - "parameters": { - "properties": { - "include": { - "description": "Glob filter, e.g. \"*.ts\"", - "type": "string" - }, - "path": { - "description": "Directory or file to search (default: .)", - "type": "string" - }, - "pattern": { - "description": "Search pattern (regex)", - "type": "string" - } - }, - "required": ["pattern"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Find files by filename pattern using shell find -name semantics. Returns matching file paths.", - "name": "glob", - "parameters": { - "properties": { - "path": { - "description": "Directory to search in (default: .)", - "type": "string" - }, - "pattern": { - "description": "Filename pattern, e.g. \"*.ts\"", - "type": "string" - } - }, - "required": ["pattern"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Delegate a focused task to a detached child agent with its own context. Use this for independent research, file exploration, or parallel work. The task returns only its final answer to this conversation. No subagents are currently defined.", - "name": "task", - "parameters": { - "properties": { - "agent": { - "description": "Declared subagent to use for the child agent", - "type": "string" - }, - "cwd": { - "description": "Working directory for the child agent. AGENTS.md and skills are discovered from here.", - "type": "string" - }, - "description": { - "description": "Short human-readable label for the delegated work", - "type": "string" - }, - "prompt": { - "description": "Focused instructions for the child agent", - "type": "string" - } - }, - "required": ["prompt"], - "type": "object" - }, - "strict": false, - "type": "function" - } - ] - } - }, - "headers": {}, - "method": "POST", - "url": "https://api.openai.com/v1/responses" - }, - "response": { - "body": { - "chunks": [ - "event: response.created\ndata: {\"type\":\"response.created\",\"response\":{\"id\":\"resp_054a16a2f2ea3217016a705b6f493c81a3a225e3a6b86bbee9\",\"object\":\"response\",\"created_at\":1785748335,\"status\":\"in_progress\",\"background\":false,\"completed_at\":null,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":null,\"max_tool_calls\":null,\"model\":\"gpt-5.4-nano-2026-03-17\",\"moderation\":null,\"output\":[],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":\"workflow:instrumentation:01KZ3E9TWTAEK4P4P3F63HKAKC::default::ta\",\"prompt_cache_retention\":\"24h\",\"reasoning\":{\"context\":\"current_turn\",\"effort\":\"none\",\"mode\":\"standard\",\"summary\":null},\"safety_identifier\":null,\"service_tier\":\"auto\",\"store\":false,\"temperature\":1,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tool_usage\":{\"image_gen\":{\"input_tokens\":0,\"input_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"output_tokens\":0,\"output_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"total_tokens\":0},\"web_search\":{\"num_requests\":0}},\"tools\":[{\"type\":\"function\",\"description\":\"Read a file or list a directory. For files, output is truncated to 2000 lines or 50KB — use offset/limit for large files. For directories, returns the list of entries.\",\"name\":\"read\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to read\"},\"offset\":{\"type\":\"number\",\"description\":\"Line number to start from (1-indexed)\"},\"limit\":{\"type\":\"number\",\"description\":\"Maximum number of lines to read\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Write content to a file. Creates the file and parent directories if they do not exist.\",\"name\":\"write\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"content\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to write\"},\"content\":{\"type\":\"string\",\"description\":\"Content to write to the file\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Edit a file using exact text replacement. The oldText must match a unique region of the file. Use replaceAll to replace all occurrences.\",\"name\":\"edit\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"oldText\",\"newText\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to edit\"},\"oldText\":{\"type\":\"string\",\"description\":\"Exact text to find (must be unique)\"},\"newText\":{\"type\":\"string\",\"description\":\"Replacement text\"},\"replaceAll\":{\"type\":\"boolean\",\"description\":\"Replace all occurrences\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Execute a bash command. Returns stdout and stderr. Output is truncated to the last 2000 lines or 50KB.\",\"name\":\"bash\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"command\"],\"properties\":{\"command\":{\"type\":\"string\",\"description\":\"Bash command to execute\"},\"timeout\":{\"type\":\"number\",\"description\":\"Timeout in seconds\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Search file contents for a regex pattern. Returns matching lines with file paths and line numbers.\",\"name\":\"grep\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Search pattern (regex)\"},\"path\":{\"type\":\"string\",\"description\":\"Directory or file to search (default: .)\"},\"include\":{\"type\":\"string\",\"description\":\"Glob filter, e.g. \\\"*.ts\\\"\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Find files by filename pattern using shell find -name semantics. Returns matching file paths.\",\"name\":\"glob\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Filename pattern, e.g. \\\"*.ts\\\"\"},\"path\":{\"type\":\"string\",\"description\":\"Directory to search in (default: .)\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Delegate a focused task to a detached child agent with its own context. Use this for independent research, file exploration, or parallel work. The task returns only its final answer to this conversation. No subagents are currently defined.\",\"name\":\"task\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"prompt\"],\"properties\":{\"description\":{\"type\":\"string\",\"description\":\"Short human-readable label for the delegated work\"},\"prompt\":{\"type\":\"string\",\"description\":\"Focused instructions for the child agent\"},\"agent\":{\"type\":\"string\",\"description\":\"Declared subagent to use for the child agent\"},\"cwd\":{\"type\":\"string\",\"description\":\"Working directory for the child agent. AGENTS.md and skills are discovered from here.\"}}},\"strict\":false}],\"top_logprobs\":0,\"top_p\":0.98,\"truncation\":\"disabled\",\"usage\":null,\"user\":null,\"metadata\":{}},\"sequence_number\":0}", - "event: response.in_progress\ndata: {\"type\":\"response.in_progress\",\"response\":{\"id\":\"resp_054a16a2f2ea3217016a705b6f493c81a3a225e3a6b86bbee9\",\"object\":\"response\",\"created_at\":1785748335,\"status\":\"in_progress\",\"background\":false,\"completed_at\":null,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":null,\"max_tool_calls\":null,\"model\":\"gpt-5.4-nano-2026-03-17\",\"moderation\":null,\"output\":[],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":\"workflow:instrumentation:01KZ3E9TWTAEK4P4P3F63HKAKC::default::ta\",\"prompt_cache_retention\":\"24h\",\"reasoning\":{\"context\":\"current_turn\",\"effort\":\"none\",\"mode\":\"standard\",\"summary\":null},\"safety_identifier\":null,\"service_tier\":\"auto\",\"store\":false,\"temperature\":1,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tool_usage\":{\"image_gen\":{\"input_tokens\":0,\"input_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"output_tokens\":0,\"output_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"total_tokens\":0},\"web_search\":{\"num_requests\":0}},\"tools\":[{\"type\":\"function\",\"description\":\"Read a file or list a directory. For files, output is truncated to 2000 lines or 50KB — use offset/limit for large files. For directories, returns the list of entries.\",\"name\":\"read\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to read\"},\"offset\":{\"type\":\"number\",\"description\":\"Line number to start from (1-indexed)\"},\"limit\":{\"type\":\"number\",\"description\":\"Maximum number of lines to read\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Write content to a file. Creates the file and parent directories if they do not exist.\",\"name\":\"write\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"content\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to write\"},\"content\":{\"type\":\"string\",\"description\":\"Content to write to the file\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Edit a file using exact text replacement. The oldText must match a unique region of the file. Use replaceAll to replace all occurrences.\",\"name\":\"edit\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"oldText\",\"newText\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to edit\"},\"oldText\":{\"type\":\"string\",\"description\":\"Exact text to find (must be unique)\"},\"newText\":{\"type\":\"string\",\"description\":\"Replacement text\"},\"replaceAll\":{\"type\":\"boolean\",\"description\":\"Replace all occurrences\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Execute a bash command. Returns stdout and stderr. Output is truncated to the last 2000 lines or 50KB.\",\"name\":\"bash\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"command\"],\"properties\":{\"command\":{\"type\":\"string\",\"description\":\"Bash command to execute\"},\"timeout\":{\"type\":\"number\",\"description\":\"Timeout in seconds\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Search file contents for a regex pattern. Returns matching lines with file paths and line numbers.\",\"name\":\"grep\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Search pattern (regex)\"},\"path\":{\"type\":\"string\",\"description\":\"Directory or file to search (default: .)\"},\"include\":{\"type\":\"string\",\"description\":\"Glob filter, e.g. \\\"*.ts\\\"\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Find files by filename pattern using shell find -name semantics. Returns matching file paths.\",\"name\":\"glob\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Filename pattern, e.g. \\\"*.ts\\\"\"},\"path\":{\"type\":\"string\",\"description\":\"Directory to search in (default: .)\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Delegate a focused task to a detached child agent with its own context. Use this for independent research, file exploration, or parallel work. The task returns only its final answer to this conversation. No subagents are currently defined.\",\"name\":\"task\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"prompt\"],\"properties\":{\"description\":{\"type\":\"string\",\"description\":\"Short human-readable label for the delegated work\"},\"prompt\":{\"type\":\"string\",\"description\":\"Focused instructions for the child agent\"},\"agent\":{\"type\":\"string\",\"description\":\"Declared subagent to use for the child agent\"},\"cwd\":{\"type\":\"string\",\"description\":\"Working directory for the child agent. AGENTS.md and skills are discovered from here.\"}}},\"strict\":false}],\"top_logprobs\":0,\"top_p\":0.98,\"truncation\":\"disabled\",\"usage\":null,\"user\":null,\"metadata\":{}},\"sequence_number\":1}", - "event: response.output_item.added\ndata: {\"type\":\"response.output_item.added\",\"item\":{\"id\":\"fc_054a16a2f2ea3217016a705b6fb64c81a38ff895c9553c8763\",\"type\":\"function_call\",\"status\":\"in_progress\",\"arguments\":\"\",\"call_id\":\"call_7fRJCCDIZe4x17WaroXZwmjV\",\"name\":\"bash\"},\"output_index\":0,\"sequence_number\":2}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"{\\\"\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6fb64c81a38ff895c9553c8763\",\"obfuscation\":\"fHq8b6uDNpdKyA\",\"output_index\":0,\"sequence_number\":3}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"command\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6fb64c81a38ff895c9553c8763\",\"obfuscation\":\"thvsscitV\",\"output_index\":0,\"sequence_number\":4}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"\\\":\\\"\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6fb64c81a38ff895c9553c8763\",\"obfuscation\":\"4KhNnIoFmUxL4\",\"output_index\":0,\"sequence_number\":5}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"cd\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6fb64c81a38ff895c9553c8763\",\"obfuscation\":\"2MAKSqM5RZc02D\",\"output_index\":0,\"sequence_number\":6}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" /\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6fb64c81a38ff895c9553c8763\",\"obfuscation\":\"rtJwB0qEFcuGP0\",\"output_index\":0,\"sequence_number\":7}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"ver\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6fb64c81a38ff895c9553c8763\",\"obfuscation\":\"F1lTsvrOqKTGW\",\"output_index\":0,\"sequence_number\":8}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"cel\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6fb64c81a38ff895c9553c8763\",\"obfuscation\":\"OW6Ho3Im2OXeD\",\"output_index\":0,\"sequence_number\":9}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/s\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6fb64c81a38ff895c9553c8763\",\"obfuscation\":\"OeETspVfDui6qp\",\"output_index\":0,\"sequence_number\":10}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"andbox\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6fb64c81a38ff895c9553c8763\",\"obfuscation\":\"TYKoEchhWA\",\"output_index\":0,\"sequence_number\":11}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6fb64c81a38ff895c9553c8763\",\"obfuscation\":\"mlWpUOMOoFsxvPZ\",\"output_index\":0,\"sequence_number\":12}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"brain\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6fb64c81a38ff895c9553c8763\",\"obfuscation\":\"dhWAHpJ9eAO\",\"output_index\":0,\"sequence_number\":13}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"trust\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6fb64c81a38ff895c9553c8763\",\"obfuscation\":\"obL0WnjlMLS\",\"output_index\":0,\"sequence_number\":14}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"data\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6fb64c81a38ff895c9553c8763\",\"obfuscation\":\"VQtHA1VRMGyD\",\"output_index\":0,\"sequence_number\":15}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6fb64c81a38ff895c9553c8763\",\"obfuscation\":\"v3crkHcTsSeS6g9\",\"output_index\":0,\"sequence_number\":16}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"brain\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6fb64c81a38ff895c9553c8763\",\"obfuscation\":\"fLyFqifDKnd\",\"output_index\":0,\"sequence_number\":17}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"trust\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6fb64c81a38ff895c9553c8763\",\"obfuscation\":\"zOZwoEc9EwU\",\"output_index\":0,\"sequence_number\":18}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"-sdk\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6fb64c81a38ff895c9553c8763\",\"obfuscation\":\"jKM1tjLK0Nox\",\"output_index\":0,\"sequence_number\":19}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"-j\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6fb64c81a38ff895c9553c8763\",\"obfuscation\":\"N8hk8Xm3OX7w2R\",\"output_index\":0,\"sequence_number\":20}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"avascript\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6fb64c81a38ff895c9553c8763\",\"obfuscation\":\"P8EgPwQ\",\"output_index\":0,\"sequence_number\":21}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/e\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6fb64c81a38ff895c9553c8763\",\"obfuscation\":\"aIWWYDGHFlbnf0\",\"output_index\":0,\"sequence_number\":22}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"2\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6fb64c81a38ff895c9553c8763\",\"obfuscation\":\"k9oYAIfrrIU8RoN\",\"output_index\":0,\"sequence_number\":23}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"e\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6fb64c81a38ff895c9553c8763\",\"obfuscation\":\"qYFO9k1cYtpeWCN\",\"output_index\":0,\"sequence_number\":24}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/sc\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6fb64c81a38ff895c9553c8763\",\"obfuscation\":\"h3KfOXIs1dENs\",\"output_index\":0,\"sequence_number\":25}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"enarios\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6fb64c81a38ff895c9553c8763\",\"obfuscation\":\"JH679LrpQ\",\"output_index\":0,\"sequence_number\":26}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/fl\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6fb64c81a38ff895c9553c8763\",\"obfuscation\":\"gMCaFOZiRoUg3\",\"output_index\":0,\"sequence_number\":27}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"ue\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6fb64c81a38ff895c9553c8763\",\"obfuscation\":\"yXl3sFNxeXVgqy\",\"output_index\":0,\"sequence_number\":28}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"-in\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6fb64c81a38ff895c9553c8763\",\"obfuscation\":\"glJkNUpBDKD1z\",\"output_index\":0,\"sequence_number\":29}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"strument\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6fb64c81a38ff895c9553c8763\",\"obfuscation\":\"VBlCVyqb\",\"output_index\":0,\"sequence_number\":30}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"ation\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6fb64c81a38ff895c9553c8763\",\"obfuscation\":\"SNMP91Q3rDW\",\"output_index\":0,\"sequence_number\":31}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" &&\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6fb64c81a38ff895c9553c8763\",\"obfuscation\":\"X3ihmCKcYuVXU\",\"output_index\":0,\"sequence_number\":32}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" node\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6fb64c81a38ff895c9553c8763\",\"obfuscation\":\"JKpFNiOkxT3\",\"output_index\":0,\"sequence_number\":33}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" scenario\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6fb64c81a38ff895c9553c8763\",\"obfuscation\":\"bFvkMJJ\",\"output_index\":0,\"sequence_number\":34}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\".cli\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6fb64c81a38ff895c9553c8763\",\"obfuscation\":\"kUiDpbheCAHH\",\"output_index\":0,\"sequence_number\":35}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\".m\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6fb64c81a38ff895c9553c8763\",\"obfuscation\":\"FxsGZq8BCGo3gl\",\"output_index\":0,\"sequence_number\":36}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"js\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6fb64c81a38ff895c9553c8763\",\"obfuscation\":\"Dp4QtMwr7c3BK1\",\"output_index\":0,\"sequence_number\":37}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" --\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6fb64c81a38ff895c9553c8763\",\"obfuscation\":\"PeegSlydvrC1L\",\"output_index\":0,\"sequence_number\":38}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"marker\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6fb64c81a38ff895c9553c8763\",\"obfuscation\":\"wvkAVwRdNi\",\"output_index\":0,\"sequence_number\":39}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" SK\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6fb64c81a38ff895c9553c8763\",\"obfuscation\":\"PTHJb9FqWdVqt\",\"output_index\":0,\"sequence_number\":40}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"ILL\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6fb64c81a38ff895c9553c8763\",\"obfuscation\":\"2fQZxFddpb4N0\",\"output_index\":0,\"sequence_number\":41}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"_DONE\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6fb64c81a38ff895c9553c8763\",\"obfuscation\":\"k70XHwhwure\",\"output_index\":0,\"sequence_number\":42}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"\\\",\\\"\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6fb64c81a38ff895c9553c8763\",\"obfuscation\":\"rGVgz0vPyy7dV\",\"output_index\":0,\"sequence_number\":43}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"timeout\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6fb64c81a38ff895c9553c8763\",\"obfuscation\":\"MAkwSc4cj\",\"output_index\":0,\"sequence_number\":44}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"\\\":\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6fb64c81a38ff895c9553c8763\",\"obfuscation\":\"RnIym4ATTjxwrH\",\"output_index\":0,\"sequence_number\":45}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"100\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6fb64c81a38ff895c9553c8763\",\"obfuscation\":\"soJ0BJsFSpsnc\",\"output_index\":0,\"sequence_number\":46}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"000\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6fb64c81a38ff895c9553c8763\",\"obfuscation\":\"DQKN9g0JhSsmY\",\"output_index\":0,\"sequence_number\":47}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"}\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6fb64c81a38ff895c9553c8763\",\"obfuscation\":\"BTOAjj33gQRbswj\",\"output_index\":0,\"sequence_number\":48}", - "event: response.function_call_arguments.done\ndata: {\"type\":\"response.function_call_arguments.done\",\"arguments\":\"{\\\"command\\\":\\\"printf SKILL_DONE\\\",\\\"timeout\\\":100000}\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6fb64c81a38ff895c9553c8763\",\"output_index\":0,\"sequence_number\":49}", - "event: response.output_item.done\ndata: {\"type\":\"response.output_item.done\",\"item\":{\"id\":\"fc_054a16a2f2ea3217016a705b6fb64c81a38ff895c9553c8763\",\"type\":\"function_call\",\"status\":\"completed\",\"arguments\":\"{\\\"command\\\":\\\"printf SKILL_DONE\\\",\\\"timeout\\\":100000}\",\"call_id\":\"call_7fRJCCDIZe4x17WaroXZwmjV\",\"name\":\"bash\"},\"output_index\":0,\"sequence_number\":50}", - "event: response.completed\ndata: {\"type\":\"response.completed\",\"response\":{\"id\":\"resp_054a16a2f2ea3217016a705b6f493c81a3a225e3a6b86bbee9\",\"object\":\"response\",\"created_at\":1785748335,\"status\":\"completed\",\"background\":false,\"completed_at\":1785748336,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":null,\"max_tool_calls\":null,\"model\":\"gpt-5.4-nano-2026-03-17\",\"moderation\":null,\"output\":[{\"id\":\"fc_054a16a2f2ea3217016a705b6fb64c81a38ff895c9553c8763\",\"type\":\"function_call\",\"status\":\"completed\",\"arguments\":\"{\\\"command\\\":\\\"printf SKILL_DONE\\\",\\\"timeout\\\":100000}\",\"call_id\":\"call_7fRJCCDIZe4x17WaroXZwmjV\",\"name\":\"bash\"}],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":\"workflow:instrumentation:01KZ3E9TWTAEK4P4P3F63HKAKC::default::ta\",\"prompt_cache_retention\":\"24h\",\"reasoning\":{\"context\":\"current_turn\",\"effort\":\"none\",\"mode\":\"standard\",\"summary\":null},\"safety_identifier\":null,\"service_tier\":\"default\",\"store\":false,\"temperature\":1,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tool_usage\":{\"image_gen\":{\"input_tokens\":0,\"input_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"output_tokens\":0,\"output_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"total_tokens\":0},\"web_search\":{\"num_requests\":0}},\"tools\":[{\"type\":\"function\",\"description\":\"Read a file or list a directory. For files, output is truncated to 2000 lines or 50KB — use offset/limit for large files. For directories, returns the list of entries.\",\"name\":\"read\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to read\"},\"offset\":{\"type\":\"number\",\"description\":\"Line number to start from (1-indexed)\"},\"limit\":{\"type\":\"number\",\"description\":\"Maximum number of lines to read\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Write content to a file. Creates the file and parent directories if they do not exist.\",\"name\":\"write\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"content\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to write\"},\"content\":{\"type\":\"string\",\"description\":\"Content to write to the file\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Edit a file using exact text replacement. The oldText must match a unique region of the file. Use replaceAll to replace all occurrences.\",\"name\":\"edit\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"oldText\",\"newText\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to edit\"},\"oldText\":{\"type\":\"string\",\"description\":\"Exact text to find (must be unique)\"},\"newText\":{\"type\":\"string\",\"description\":\"Replacement text\"},\"replaceAll\":{\"type\":\"boolean\",\"description\":\"Replace all occurrences\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Execute a bash command. Returns stdout and stderr. Output is truncated to the last 2000 lines or 50KB.\",\"name\":\"bash\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"command\"],\"properties\":{\"command\":{\"type\":\"string\",\"description\":\"Bash command to execute\"},\"timeout\":{\"type\":\"number\",\"description\":\"Timeout in seconds\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Search file contents for a regex pattern. Returns matching lines with file paths and line numbers.\",\"name\":\"grep\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Search pattern (regex)\"},\"path\":{\"type\":\"string\",\"description\":\"Directory or file to search (default: .)\"},\"include\":{\"type\":\"string\",\"description\":\"Glob filter, e.g. \\\"*.ts\\\"\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Find files by filename pattern using shell find -name semantics. Returns matching file paths.\",\"name\":\"glob\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Filename pattern, e.g. \\\"*.ts\\\"\"},\"path\":{\"type\":\"string\",\"description\":\"Directory to search in (default: .)\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Delegate a focused task to a detached child agent with its own context. Use this for independent research, file exploration, or parallel work. The task returns only its final answer to this conversation. No subagents are currently defined.\",\"name\":\"task\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"prompt\"],\"properties\":{\"description\":{\"type\":\"string\",\"description\":\"Short human-readable label for the delegated work\"},\"prompt\":{\"type\":\"string\",\"description\":\"Focused instructions for the child agent\"},\"agent\":{\"type\":\"string\",\"description\":\"Declared subagent to use for the child agent\"},\"cwd\":{\"type\":\"string\",\"description\":\"Working directory for the child agent. AGENTS.md and skills are discovered from here.\"}}},\"strict\":false}],\"top_logprobs\":0,\"top_p\":0.98,\"truncation\":\"disabled\",\"usage\":{\"input_tokens\":3421,\"input_tokens_details\":{\"cache_write_tokens\":0,\"cached_tokens\":1536},\"output_tokens\":59,\"output_tokens_details\":{\"reasoning_tokens\":0},\"total_tokens\":3480},\"user\":null,\"metadata\":{}},\"sequence_number\":51}" - ], - "kind": "sse" - }, - "headers": { - "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", - "alt-svc": "h3=\":443\"; ma=86400", - "cf-cache-status": "DYNAMIC", - "cf-ray": "a25433173b9c7bb6-IAD", - "connection": "keep-alive", - "content-type": "text/event-stream; charset=utf-8", - "date": "Mon, 03 Aug 2026 09:12:15 GMT", - "openai-organization": "braintrust-data", - "openai-processing-ms": "200", - "openai-project": "proj_vsCSXafhhByzWOThMrJcZiw9", - "openai-version": "2020-10-01", - "server": "cloudflare", - "set-cookie": "[REDACTED]", - "strict-transport-security": "max-age=31536000; includeSubDomains; preload", - "transfer-encoding": "chunked", - "x-content-type-options": "nosniff", - "x-ratelimit-limit-requests": "30000", - "x-ratelimit-limit-tokens": "180000000", - "x-ratelimit-remaining-requests": "29999", - "x-ratelimit-remaining-tokens": "179996124", - "x-ratelimit-reset-requests": "2ms", - "x-ratelimit-reset-tokens": "1ms", - "x-request-id": "req_3d947e9c4a9c4ea6a016b86fe1407743" - }, - "status": 200, - "statusText": "OK" - } - }, - { - "callIndex": 14, - "id": "bc08e707f2920073", - "matchKey": "POST api.openai.com/v1/responses", - "recordedAt": "2026-08-03T09:12:16.776Z", - "request": { - "body": { - "kind": "json", - "value": { - "input": [ - { - "content": "You are running in headless mode with no human operator. Work autonomously — never ask questions, never wait for user input. Make your best judgment and proceed independently.\n\nYou are a deterministic Flue instrumentation test agent. Follow user instructions exactly. When asked for a marker, output only that marker and no extra text. When running a local skill file, read it yourself and do not delegate it to a task.\n\n## Available Skills\n\nThe following skills provide specialized instructions for specific tasks. When a task matches a skill description, activate that skill before proceeding so its full instructions are loaded. Skill instructions and supporting resources stay lazy until activation or explicit file reads.\n\n- **e2e-flue-skill** — Deterministic skill used by the Braintrust Flue instrumentation e2e test.\n\nDate: Mon, Aug 3, 2026\nWorking directory: .\n\nDirectory structure:\n.agents\n.flue\n.flue-build\n__cassettes__\n__snapshots__\nassertions.ts\ncassette-filter.mjs\nconstants.mjs\ndist\nnode_modules\npackage.json\npnpm-lock.yaml\nscenario.cli.mjs\nscenario.impl.mjs\nscenario.mjs\nscenario.test.ts\nscenario.ts\nscenario.v2.ts\nv2-assertions.ts\nversions", - "role": "developer" - }, - { - "content": [ - { - "text": "Run the skill named \"e2e-flue-skill\" with arguments {\"marker\":\"SKILL_DONE\"}. Output only the marker and no extra text when finished.", - "type": "input_text" - } - ], - "role": "user" - }, - { - "arguments": "{\"description\":\"Run e2e-flue-skill with marker SKILL_DONE\",\"prompt\":\"Run the skill named \\\"e2e-flue-skill\\\" with arguments {\\\"marker\\\":\\\"SKILL_DONE\\\"}. Ensure completion and report the final output marker only.\",\"agent\":\"e2e-flue-skill\",\"cwd\":\".\"}", - "call_id": "call_w6pyYZgnhlSOAXul75DAG1Cj", - "id": "fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6", - "name": "task", - "type": "function_call" - }, - { - "call_id": "call_w6pyYZgnhlSOAXul75DAG1Cj", - "output": "[flue] Subagent \"e2e-flue-skill\" is not declared. Available: (none).", - "type": "function_call_output" - }, - { - "arguments": "{\"command\":\"find . -maxdepth 4 -name AGENTS.md -print\",\"timeout\":100000}", - "call_id": "call_6zEBnJRmXFIkGFwNjWLZP4uC", - "id": "fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee", - "name": "bash", - "type": "function_call" - }, - { - "call_id": "call_6zEBnJRmXFIkGFwNjWLZP4uC", - "output": "total 4\ndrwxr-xr-x 3 vercel-sandbox vercel-sandbox 20 Aug 3 09:10 .\ndrwxr-xr-x 10 vercel-sandbox vercel-sandbox 4096 Aug 3 09:11 ..\ndrwxr-xr-x 3 vercel-sandbox vercel-sandbox 28 Aug 3 09:10 skills", - "type": "function_call_output" - }, - { - "arguments": "{\"command\":\"find . -maxdepth 4 -type f -name AGENTS.md -print -exec sed -n '1,200p' {} \\\\;\",\"timeout\":100000}", - "call_id": "call_jQUCie0ZgaXUrZCDcezp0msl", - "id": "fc_054a16a2f2ea3217016a705b67575881a3aacde02b9053a321", - "name": "bash", - "type": "function_call" - }, - { - "call_id": "call_jQUCie0ZgaXUrZCDcezp0msl", - "output": "(no output)", - "type": "function_call_output" - }, - { - "arguments": "{\"command\":\"find . -maxdepth 6 -type f -name 'AGENTS.md' -print\",\"timeout\":100000}", - "call_id": "call_NgMSUb6wIR0GAf9U8LXERtBQ", - "id": "fc_054a16a2f2ea3217016a705b68c39881a3b1e518599236e41a", - "name": "bash", - "type": "function_call" - }, - { - "call_id": "call_NgMSUb6wIR0GAf9U8LXERtBQ", - "output": "(no output)", - "type": "function_call_output" - }, - { - "arguments": "{\"command\":\"find . -maxdepth 5 -type f -name '*flue*skill*' -o -name '*skill*' | head\",\"timeout\":100000}", - "call_id": "call_BXKiQ5KG8c42XPPp94zgcY3H", - "id": "fc_054a16a2f2ea3217016a705b69edd081a38c31583ec56eca5f", - "name": "bash", - "type": "function_call" - }, - { - "call_id": "call_BXKiQ5KG8c42XPPp94zgcY3H", - "output": "./.agents/skills\n./.agents/skills/e2e-flue-skill", - "type": "function_call_output" - }, - { - "arguments": "{\"command\":\"find ./.agents/skills/e2e-flue-skill -maxdepth 2 -type f -print\",\"timeout\":100000}", - "call_id": "call_Zmmrs64LFscpAHsIUaIfEcAN", - "id": "fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63", - "name": "bash", - "type": "function_call" - }, - { - "call_id": "call_Zmmrs64LFscpAHsIUaIfEcAN", - "output": "total 4\ndrwxr-xr-x 2 vercel-sandbox vercel-sandbox 22 Aug 3 07:55 .\ndrwxr-xr-x 3 vercel-sandbox vercel-sandbox 28 Aug 3 07:55 ..\n-rw-r--r-- 1 vercel-sandbox vercel-sandbox 194 Aug 3 07:55 SKILL.md\n./.agents/skills/e2e-flue-skill/SKILL.md", - "type": "function_call_output" - }, - { - "arguments": "{\"path\":\"./.agents/skills/e2e-flue-skill/SKILL.md\",\"offset\":1,\"limit\":200}", - "call_id": "call_KXcpmIy4Cbfugk4SNT2Ge7sG", - "id": "fc_054a16a2f2ea3217016a705b6d71cc81a38ac06634a309a997", - "name": "read", - "type": "function_call" - }, - { - "call_id": "call_KXcpmIy4Cbfugk4SNT2Ge7sG", - "output": "---\nname: e2e-flue-skill\ndescription: Deterministic skill used by the Braintrust Flue instrumentation e2e test.\n---\n\nReturn the marker from the provided args exactly once. Output no other text.\n", - "type": "function_call_output" - }, - { - "arguments": "{\"command\":\"cat ./package.json\",\"timeout\":100000}", - "call_id": "call_xQybhnQYuflmfp7gyAScVKjY", - "id": "fc_054a16a2f2ea3217016a705b6e90f481a39bc44496fbadf441", - "name": "bash", - "type": "function_call" - }, - { - "call_id": "call_xQybhnQYuflmfp7gyAScVKjY", - "output": "v24.14.1\ntotal 244\ndrwxr-xr-x 7 vercel-sandbox vercel-sandbox 4096 Aug 3 09:06 .\ndrwxr-xr-x 46 vercel-sandbox vercel-sandbox 4096 Aug 3 07:55 ..\ndrwxr-xr-x 3 vercel-sandbox vercel-sandbox 20 Aug 3 07:55 .agents\ndrwxr-xr-x 3 vercel-sandbox vercel-sandbox 38 Aug 3 07:55 .flue\ndrwxr-xr-x 2 vercel-sandbox vercel-sandbox 218 Aug 3 09:11 __cassettes__\ndrwxr-xr-x 2 vercel-sandbox vercel-sandbox 4096 Aug 3 08:49 __snapshots__\n-rw-r--r-- 1 vercel-sandbox vercel-sandbox 12484 Aug 3 08:45 assertions.ts\n-rw-r--r-- 1 vercel-sandbox vercel-sandbox 380 Aug 3 07:55 cassette-filter.mjs\n-rw-r--r-- 1 vercel-sandbox vercel-sandbox 215 Aug 3 07:55 constants.mjs\n-rw-r--r-- 1 vercel-sandbox vercel-sandbox 2000 Aug 3 09:06 package.json\n-rw-r--r-- 1 vercel-sandbox vercel-sandbox 167098 Aug 3 09:06 pnpm-lock.yaml\n-rw-r--r-- 1 vercel-sandbox vercel-sandbox 123 Aug 3 07:55 scenario.cli.mjs\n-rw-r--r-- 1 vercel-sandbox vercel-sandbox 6877 Aug 3 09:06 scenario.impl.mjs\n-rw-r--r-- 1 vercel-sandbox vercel-sandbox 203 Aug 3 07:55 scenario.mjs\n-rw-r--r-- 1 vercel-sandbox vercel-sandbox 8066 Aug 3 09:09 scenario.test.ts\n-rw-r--r-- 1 vercel-sandbox vercel-sandbox 121 Aug 3 07:55 scenario.ts\n-rw-r--r-- 1 vercel-sandbox vercel-sandbox 2376 Aug 3 08:49 scenario.v2.ts\n-rw-r--r-- 1 vercel-sandbox vercel-sandbox 4616 Aug 3 08:50 v2-assertions.ts\ndrwxr-xr-x 4 vercel-sandbox vercel-sandbox 26 Aug 3 08:06 versions\n{\n \"name\": \"flue-instrumentation-scenario\",\n \"private\": true,\n \"type\": \"module\",\n \"braintrustScenario\": {\n \"bump\": {\n \"dependencies\": {\n \"flue-cli-v0-8-latest\": {\n \"package\": \"@flue/cli\",\n \"range\": \">=0.8.0 <0.9.0\"\n },\n \"flue-cli-v1-latest\": {\n \"package\": \"@flue/cli\",\n \"range\": \">=1.0.0-beta.3 <2.0.0-0\",\n \"allowPrerelease\": true\n },\n \"flue-runtime-v0-8-latest\": {\n \"package\": \"@flue/runtime\",\n \"range\": \">=0.8.0 <0.9.0\"\n },\n \"flue-runtime-v1-latest\": {\n \"package\": \"@flue/runtime\",\n \"range\": \">=1.0.0-beta.3 <2.0.0-0\",\n \"allowPrerelease\": true\n },\n \"flue-runtime-v2-latest\": {\n \"package\": \"@flue/runtime\",\n \"range\": \">=2.0.0 <3.0.0\"\n }\n }\n }\n },\n \"dependencies\": {\n \"@flue/cli\": \"0.8.0\",\n \"@flue/runtime\": \"0.8.0\",\n \"flue-cli-v0-8-latest\": \"npm:@flue/cli@0.8.1\",\n \"flue-cli-v1\": \"npm:@flue/cli@1.0.0-beta.3\",\n \"flue-cli-v1-latest\": \"npm:@flue/cli@1.0.0-beta.9\",\n \"flue-runtime-v0-8-latest\": \"npm:@flue/runtime@0.8.1\",\n \"flue-runtime-v1\": \"npm:@flue/runtime@1.0.0-beta.3\",\n \"flue-runtime-v1-latest\": \"npm:@flue/runtime@1.0.0-beta.9\",\n \"flue-runtime-v2\": \"npm:@flue/runtime@2.0.0\",\n \"flue-runtime-v2-latest\": \"npm:@flue/runtime@2.0.1\",\n \"pi-ai-v2\": \"npm:@earendil-works/pi-ai@0.83.0\",\n \"valibot\": \"1.4.2\"\n },\n \"pnpm\": {\n \"overrides\": {\n \"@flue/runtime@0.8.0>@earendil-works/pi-agent-core\": \"0.75.4\",\n \"@flue/runtime@0.8.0>@earendil-works/pi-ai\": \"0.75.4\",\n \"@flue/runtime@0.8.1>@earendil-works/pi-agent-core\": \"0.75.4\",\n \"@flue/runtime@0.8.1>@earendil-works/pi-ai\": \"0.75.4\",\n \"@hono/node-server@>=2.0.0 <2.0.10\": \"^2.0.10\",\n \"fast-uri@<3.1.4\": \"^3.1.4\",\n \"hono@>=4.0.0 <4.12.27\": \"^4.12.27\",\n \"brace-expansion@>=3.0.0 <5.0.8\": \"^5.0.8\",\n \"postcss@<8.5.18\": \"^8.5.18\",\n \"valibot@<1.4.2\": \"^1.4.2\"\n }\n }\n}", - "type": "function_call_output" - }, - { - "arguments": "{\"command\":\"printf SKILL_DONE\",\"timeout\":100000}", - "call_id": "call_7fRJCCDIZe4x17WaroXZwmjV", - "id": "fc_054a16a2f2ea3217016a705b6fb64c81a38ff895c9553c8763", - "name": "bash", - "type": "function_call" - }, - { - "call_id": "call_7fRJCCDIZe4x17WaroXZwmjV", - "output": "node:internal/modules/package_json_reader:301\n throw new ERR_MODULE_NOT_FOUND(packageName, fileURLToPath(base), null);\n ^\n\nError [ERR_MODULE_NOT_FOUND]: Cannot find package '@flue/cli' imported from ./scenario.impl.mjs\n at Object.getPackageJSONURL (node:internal/modules/package_json_reader:301:9)\n at packageResolve (node:internal/modules/esm/resolve:768:81)\n at moduleResolve (node:internal/modules/esm/resolve:859:18)\n at defaultResolve (node:internal/modules/esm/resolve:991:11)\n at #cachedDefaultResolve (node:internal/modules/esm/loader:719:20)\n at #resolveAndMaybeBlockOnLoaderThread (node:internal/modules/esm/loader:736:38)\n at ModuleLoader.resolveSync (node:internal/modules/esm/loader:765:52)\n at Object.resolve (node:internal/modules/esm/initialize_import_meta:37:25)\n at file://./scenario.impl.mjs:12:33\n at ModuleJob.run (node:internal/modules/esm/module_job:430:25) {\n code: 'ERR_MODULE_NOT_FOUND'\n}\n\nNode.js v24.14.1\n\nCommand exited with code 1", - "type": "function_call_output" - } - ], - "model": "gpt-5.4-nano", - "prompt_cache_key": "[REDACTED]", - "reasoning": { - "effort": "none" - }, - "store": false, - "stream": true, - "tools": [ - { - "description": "Read a file or list a directory. For files, output is truncated to 2000 lines or 50KB — use offset/limit for large files. For directories, returns the list of entries.", - "name": "read", - "parameters": { - "properties": { - "limit": { - "description": "Maximum number of lines to read", - "type": "number" - }, - "offset": { - "description": "Line number to start from (1-indexed)", - "type": "number" - }, - "path": { - "description": "Path to the file to read", - "type": "string" - } - }, - "required": ["path"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Write content to a file. Creates the file and parent directories if they do not exist.", - "name": "write", - "parameters": { - "properties": { - "content": { - "description": "Content to write to the file", - "type": "string" - }, - "path": { - "description": "Path to the file to write", - "type": "string" - } - }, - "required": ["path", "content"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Edit a file using exact text replacement. The oldText must match a unique region of the file. Use replaceAll to replace all occurrences.", - "name": "edit", - "parameters": { - "properties": { - "newText": { - "description": "Replacement text", - "type": "string" - }, - "oldText": { - "description": "Exact text to find (must be unique)", - "type": "string" - }, - "path": { - "description": "Path to the file to edit", - "type": "string" - }, - "replaceAll": { - "description": "Replace all occurrences", - "type": "boolean" - } - }, - "required": ["path", "oldText", "newText"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Execute a bash command. Returns stdout and stderr. Output is truncated to the last 2000 lines or 50KB.", - "name": "bash", - "parameters": { - "properties": { - "command": { - "description": "Bash command to execute", - "type": "string" - }, - "timeout": { - "description": "Timeout in seconds", - "type": "number" - } - }, - "required": ["command"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Search file contents for a regex pattern. Returns matching lines with file paths and line numbers.", - "name": "grep", - "parameters": { - "properties": { - "include": { - "description": "Glob filter, e.g. \"*.ts\"", - "type": "string" - }, - "path": { - "description": "Directory or file to search (default: .)", - "type": "string" - }, - "pattern": { - "description": "Search pattern (regex)", - "type": "string" - } - }, - "required": ["pattern"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Find files by filename pattern using shell find -name semantics. Returns matching file paths.", - "name": "glob", - "parameters": { - "properties": { - "path": { - "description": "Directory to search in (default: .)", - "type": "string" - }, - "pattern": { - "description": "Filename pattern, e.g. \"*.ts\"", - "type": "string" - } - }, - "required": ["pattern"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Delegate a focused task to a detached child agent with its own context. Use this for independent research, file exploration, or parallel work. The task returns only its final answer to this conversation. No subagents are currently defined.", - "name": "task", - "parameters": { - "properties": { - "agent": { - "description": "Declared subagent to use for the child agent", - "type": "string" - }, - "cwd": { - "description": "Working directory for the child agent. AGENTS.md and skills are discovered from here.", - "type": "string" - }, - "description": { - "description": "Short human-readable label for the delegated work", - "type": "string" - }, - "prompt": { - "description": "Focused instructions for the child agent", - "type": "string" - } - }, - "required": ["prompt"], - "type": "object" - }, - "strict": false, - "type": "function" - } - ] - } - }, - "headers": {}, - "method": "POST", - "url": "https://api.openai.com/v1/responses" - }, - "response": { - "body": { - "chunks": [ - "event: response.created\ndata: {\"type\":\"response.created\",\"response\":{\"id\":\"resp_054a16a2f2ea3217016a705b70474881a39a050b8889ca15cd\",\"object\":\"response\",\"created_at\":1785748336,\"status\":\"in_progress\",\"background\":false,\"completed_at\":null,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":null,\"max_tool_calls\":null,\"model\":\"gpt-5.4-nano-2026-03-17\",\"moderation\":null,\"output\":[],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":\"workflow:instrumentation:01KZ3E9TWTAEK4P4P3F63HKAKC::default::ta\",\"prompt_cache_retention\":\"24h\",\"reasoning\":{\"context\":\"current_turn\",\"effort\":\"none\",\"mode\":\"standard\",\"summary\":null},\"safety_identifier\":null,\"service_tier\":\"auto\",\"store\":false,\"temperature\":1,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tool_usage\":{\"image_gen\":{\"input_tokens\":0,\"input_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"output_tokens\":0,\"output_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"total_tokens\":0},\"web_search\":{\"num_requests\":0}},\"tools\":[{\"type\":\"function\",\"description\":\"Read a file or list a directory. For files, output is truncated to 2000 lines or 50KB — use offset/limit for large files. For directories, returns the list of entries.\",\"name\":\"read\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to read\"},\"offset\":{\"type\":\"number\",\"description\":\"Line number to start from (1-indexed)\"},\"limit\":{\"type\":\"number\",\"description\":\"Maximum number of lines to read\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Write content to a file. Creates the file and parent directories if they do not exist.\",\"name\":\"write\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"content\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to write\"},\"content\":{\"type\":\"string\",\"description\":\"Content to write to the file\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Edit a file using exact text replacement. The oldText must match a unique region of the file. Use replaceAll to replace all occurrences.\",\"name\":\"edit\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"oldText\",\"newText\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to edit\"},\"oldText\":{\"type\":\"string\",\"description\":\"Exact text to find (must be unique)\"},\"newText\":{\"type\":\"string\",\"description\":\"Replacement text\"},\"replaceAll\":{\"type\":\"boolean\",\"description\":\"Replace all occurrences\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Execute a bash command. Returns stdout and stderr. Output is truncated to the last 2000 lines or 50KB.\",\"name\":\"bash\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"command\"],\"properties\":{\"command\":{\"type\":\"string\",\"description\":\"Bash command to execute\"},\"timeout\":{\"type\":\"number\",\"description\":\"Timeout in seconds\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Search file contents for a regex pattern. Returns matching lines with file paths and line numbers.\",\"name\":\"grep\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Search pattern (regex)\"},\"path\":{\"type\":\"string\",\"description\":\"Directory or file to search (default: .)\"},\"include\":{\"type\":\"string\",\"description\":\"Glob filter, e.g. \\\"*.ts\\\"\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Find files by filename pattern using shell find -name semantics. Returns matching file paths.\",\"name\":\"glob\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Filename pattern, e.g. \\\"*.ts\\\"\"},\"path\":{\"type\":\"string\",\"description\":\"Directory to search in (default: .)\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Delegate a focused task to a detached child agent with its own context. Use this for independent research, file exploration, or parallel work. The task returns only its final answer to this conversation. No subagents are currently defined.\",\"name\":\"task\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"prompt\"],\"properties\":{\"description\":{\"type\":\"string\",\"description\":\"Short human-readable label for the delegated work\"},\"prompt\":{\"type\":\"string\",\"description\":\"Focused instructions for the child agent\"},\"agent\":{\"type\":\"string\",\"description\":\"Declared subagent to use for the child agent\"},\"cwd\":{\"type\":\"string\",\"description\":\"Working directory for the child agent. AGENTS.md and skills are discovered from here.\"}}},\"strict\":false}],\"top_logprobs\":0,\"top_p\":0.98,\"truncation\":\"disabled\",\"usage\":null,\"user\":null,\"metadata\":{}},\"sequence_number\":0}", - "event: response.in_progress\ndata: {\"type\":\"response.in_progress\",\"response\":{\"id\":\"resp_054a16a2f2ea3217016a705b70474881a39a050b8889ca15cd\",\"object\":\"response\",\"created_at\":1785748336,\"status\":\"in_progress\",\"background\":false,\"completed_at\":null,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":null,\"max_tool_calls\":null,\"model\":\"gpt-5.4-nano-2026-03-17\",\"moderation\":null,\"output\":[],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":\"workflow:instrumentation:01KZ3E9TWTAEK4P4P3F63HKAKC::default::ta\",\"prompt_cache_retention\":\"24h\",\"reasoning\":{\"context\":\"current_turn\",\"effort\":\"none\",\"mode\":\"standard\",\"summary\":null},\"safety_identifier\":null,\"service_tier\":\"auto\",\"store\":false,\"temperature\":1,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tool_usage\":{\"image_gen\":{\"input_tokens\":0,\"input_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"output_tokens\":0,\"output_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"total_tokens\":0},\"web_search\":{\"num_requests\":0}},\"tools\":[{\"type\":\"function\",\"description\":\"Read a file or list a directory. For files, output is truncated to 2000 lines or 50KB — use offset/limit for large files. For directories, returns the list of entries.\",\"name\":\"read\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to read\"},\"offset\":{\"type\":\"number\",\"description\":\"Line number to start from (1-indexed)\"},\"limit\":{\"type\":\"number\",\"description\":\"Maximum number of lines to read\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Write content to a file. Creates the file and parent directories if they do not exist.\",\"name\":\"write\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"content\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to write\"},\"content\":{\"type\":\"string\",\"description\":\"Content to write to the file\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Edit a file using exact text replacement. The oldText must match a unique region of the file. Use replaceAll to replace all occurrences.\",\"name\":\"edit\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"oldText\",\"newText\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to edit\"},\"oldText\":{\"type\":\"string\",\"description\":\"Exact text to find (must be unique)\"},\"newText\":{\"type\":\"string\",\"description\":\"Replacement text\"},\"replaceAll\":{\"type\":\"boolean\",\"description\":\"Replace all occurrences\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Execute a bash command. Returns stdout and stderr. Output is truncated to the last 2000 lines or 50KB.\",\"name\":\"bash\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"command\"],\"properties\":{\"command\":{\"type\":\"string\",\"description\":\"Bash command to execute\"},\"timeout\":{\"type\":\"number\",\"description\":\"Timeout in seconds\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Search file contents for a regex pattern. Returns matching lines with file paths and line numbers.\",\"name\":\"grep\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Search pattern (regex)\"},\"path\":{\"type\":\"string\",\"description\":\"Directory or file to search (default: .)\"},\"include\":{\"type\":\"string\",\"description\":\"Glob filter, e.g. \\\"*.ts\\\"\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Find files by filename pattern using shell find -name semantics. Returns matching file paths.\",\"name\":\"glob\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Filename pattern, e.g. \\\"*.ts\\\"\"},\"path\":{\"type\":\"string\",\"description\":\"Directory to search in (default: .)\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Delegate a focused task to a detached child agent with its own context. Use this for independent research, file exploration, or parallel work. The task returns only its final answer to this conversation. No subagents are currently defined.\",\"name\":\"task\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"prompt\"],\"properties\":{\"description\":{\"type\":\"string\",\"description\":\"Short human-readable label for the delegated work\"},\"prompt\":{\"type\":\"string\",\"description\":\"Focused instructions for the child agent\"},\"agent\":{\"type\":\"string\",\"description\":\"Declared subagent to use for the child agent\"},\"cwd\":{\"type\":\"string\",\"description\":\"Working directory for the child agent. AGENTS.md and skills are discovered from here.\"}}},\"strict\":false}],\"top_logprobs\":0,\"top_p\":0.98,\"truncation\":\"disabled\",\"usage\":null,\"user\":null,\"metadata\":{}},\"sequence_number\":1}", - "event: response.output_item.added\ndata: {\"type\":\"response.output_item.added\",\"item\":{\"id\":\"msg_054a16a2f2ea3217016a705b70a41481a38ece40ecddf69512\",\"type\":\"message\",\"status\":\"in_progress\",\"content\":[],\"phase\":\"final_answer\",\"role\":\"assistant\"},\"output_index\":0,\"sequence_number\":2}", - "event: response.content_part.added\ndata: {\"type\":\"response.content_part.added\",\"content_index\":0,\"item_id\":\"msg_054a16a2f2ea3217016a705b70a41481a38ece40ecddf69512\",\"output_index\":0,\"part\":{\"type\":\"output_text\",\"annotations\":[],\"logprobs\":[],\"text\":\"\"},\"sequence_number\":3}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"SK\",\"item_id\":\"msg_054a16a2f2ea3217016a705b70a41481a38ece40ecddf69512\",\"logprobs\":[],\"obfuscation\":\"M9SWfj095W22jm\",\"output_index\":0,\"sequence_number\":4}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"ILL\",\"item_id\":\"msg_054a16a2f2ea3217016a705b70a41481a38ece40ecddf69512\",\"logprobs\":[],\"obfuscation\":\"NnKtLGfgvufIN\",\"output_index\":0,\"sequence_number\":5}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"_DONE\",\"item_id\":\"msg_054a16a2f2ea3217016a705b70a41481a38ece40ecddf69512\",\"logprobs\":[],\"obfuscation\":\"wHpr2r6Jc26\",\"output_index\":0,\"sequence_number\":6}", - "event: response.output_text.done\ndata: {\"type\":\"response.output_text.done\",\"content_index\":0,\"item_id\":\"msg_054a16a2f2ea3217016a705b70a41481a38ece40ecddf69512\",\"logprobs\":[],\"output_index\":0,\"sequence_number\":7,\"text\":\"SKILL_DONE\"}", - "event: response.content_part.done\ndata: {\"type\":\"response.content_part.done\",\"content_index\":0,\"item_id\":\"msg_054a16a2f2ea3217016a705b70a41481a38ece40ecddf69512\",\"output_index\":0,\"part\":{\"type\":\"output_text\",\"annotations\":[],\"logprobs\":[],\"text\":\"SKILL_DONE\"},\"sequence_number\":8}", - "event: response.output_item.done\ndata: {\"type\":\"response.output_item.done\",\"item\":{\"id\":\"msg_054a16a2f2ea3217016a705b70a41481a38ece40ecddf69512\",\"type\":\"message\",\"status\":\"completed\",\"content\":[{\"type\":\"output_text\",\"annotations\":[],\"logprobs\":[],\"text\":\"SKILL_DONE\"}],\"phase\":\"final_answer\",\"role\":\"assistant\"},\"output_index\":0,\"sequence_number\":9}", - "event: response.completed\ndata: {\"type\":\"response.completed\",\"response\":{\"id\":\"resp_054a16a2f2ea3217016a705b70474881a39a050b8889ca15cd\",\"object\":\"response\",\"created_at\":1785748336,\"status\":\"completed\",\"background\":false,\"completed_at\":1785748336,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":null,\"max_tool_calls\":null,\"model\":\"gpt-5.4-nano-2026-03-17\",\"moderation\":null,\"output\":[{\"id\":\"msg_054a16a2f2ea3217016a705b70a41481a38ece40ecddf69512\",\"type\":\"message\",\"status\":\"completed\",\"content\":[{\"type\":\"output_text\",\"annotations\":[],\"logprobs\":[],\"text\":\"SKILL_DONE\"}],\"phase\":\"final_answer\",\"role\":\"assistant\"}],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":\"workflow:instrumentation:01KZ3E9TWTAEK4P4P3F63HKAKC::default::ta\",\"prompt_cache_retention\":\"24h\",\"reasoning\":{\"context\":\"current_turn\",\"effort\":\"none\",\"mode\":\"standard\",\"summary\":null},\"safety_identifier\":null,\"service_tier\":\"default\",\"store\":false,\"temperature\":1,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tool_usage\":{\"image_gen\":{\"input_tokens\":0,\"input_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"output_tokens\":0,\"output_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"total_tokens\":0},\"web_search\":{\"num_requests\":0}},\"tools\":[{\"type\":\"function\",\"description\":\"Read a file or list a directory. For files, output is truncated to 2000 lines or 50KB — use offset/limit for large files. For directories, returns the list of entries.\",\"name\":\"read\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to read\"},\"offset\":{\"type\":\"number\",\"description\":\"Line number to start from (1-indexed)\"},\"limit\":{\"type\":\"number\",\"description\":\"Maximum number of lines to read\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Write content to a file. Creates the file and parent directories if they do not exist.\",\"name\":\"write\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"content\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to write\"},\"content\":{\"type\":\"string\",\"description\":\"Content to write to the file\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Edit a file using exact text replacement. The oldText must match a unique region of the file. Use replaceAll to replace all occurrences.\",\"name\":\"edit\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"oldText\",\"newText\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to edit\"},\"oldText\":{\"type\":\"string\",\"description\":\"Exact text to find (must be unique)\"},\"newText\":{\"type\":\"string\",\"description\":\"Replacement text\"},\"replaceAll\":{\"type\":\"boolean\",\"description\":\"Replace all occurrences\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Execute a bash command. Returns stdout and stderr. Output is truncated to the last 2000 lines or 50KB.\",\"name\":\"bash\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"command\"],\"properties\":{\"command\":{\"type\":\"string\",\"description\":\"Bash command to execute\"},\"timeout\":{\"type\":\"number\",\"description\":\"Timeout in seconds\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Search file contents for a regex pattern. Returns matching lines with file paths and line numbers.\",\"name\":\"grep\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Search pattern (regex)\"},\"path\":{\"type\":\"string\",\"description\":\"Directory or file to search (default: .)\"},\"include\":{\"type\":\"string\",\"description\":\"Glob filter, e.g. \\\"*.ts\\\"\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Find files by filename pattern using shell find -name semantics. Returns matching file paths.\",\"name\":\"glob\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Filename pattern, e.g. \\\"*.ts\\\"\"},\"path\":{\"type\":\"string\",\"description\":\"Directory to search in (default: .)\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Delegate a focused task to a detached child agent with its own context. Use this for independent research, file exploration, or parallel work. The task returns only its final answer to this conversation. No subagents are currently defined.\",\"name\":\"task\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"prompt\"],\"properties\":{\"description\":{\"type\":\"string\",\"description\":\"Short human-readable label for the delegated work\"},\"prompt\":{\"type\":\"string\",\"description\":\"Focused instructions for the child agent\"},\"agent\":{\"type\":\"string\",\"description\":\"Declared subagent to use for the child agent\"},\"cwd\":{\"type\":\"string\",\"description\":\"Working directory for the child agent. AGENTS.md and skills are discovered from here.\"}}},\"strict\":false}],\"top_logprobs\":0,\"top_p\":0.98,\"truncation\":\"disabled\",\"usage\":{\"input_tokens\":3812,\"input_tokens_details\":{\"cache_write_tokens\":0,\"cached_tokens\":2560},\"output_tokens\":7,\"output_tokens_details\":{\"reasoning_tokens\":0},\"total_tokens\":3819},\"user\":null,\"metadata\":{}},\"sequence_number\":10}" - ], - "kind": "sse" - }, - "headers": { - "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", - "alt-svc": "h3=\":443\"; ma=86400", - "cf-cache-status": "DYNAMIC", - "cf-ray": "a254331d7fe07bb6-IAD", - "connection": "keep-alive", - "content-type": "text/event-stream; charset=utf-8", - "date": "Mon, 03 Aug 2026 09:12:16 GMT", - "openai-organization": "braintrust-data", - "openai-processing-ms": "201", - "openai-project": "proj_vsCSXafhhByzWOThMrJcZiw9", - "openai-version": "2020-10-01", - "server": "cloudflare", - "set-cookie": "[REDACTED]", - "strict-transport-security": "max-age=31536000; includeSubDomains; preload", - "transfer-encoding": "chunked", - "x-content-type-options": "nosniff", - "x-ratelimit-limit-requests": "30000", - "x-ratelimit-limit-tokens": "180000000", - "x-ratelimit-remaining-requests": "29999", - "x-ratelimit-remaining-tokens": "179995734", - "x-ratelimit-reset-requests": "2ms", - "x-ratelimit-reset-tokens": "1ms", - "x-request-id": "req_6dc67fd2f6124704a9d86e5e918bac37" - }, - "status": 200, - "statusText": "OK" - } - }, - { - "callIndex": 15, - "id": "b3f219cac4c52d6c", - "matchKey": "POST api.openai.com/v1/responses", - "recordedAt": "2026-08-03T09:12:17.515Z", - "request": { - "body": { - "kind": "json", - "value": { - "input": [ - { - "content": "You are running in headless mode with no human operator. Work autonomously — never ask questions, never wait for user input. Make your best judgment and proceed independently.\n\nYou are a deterministic Flue instrumentation test agent. Follow user instructions exactly. When asked for a marker, output only that marker and no extra text. When running a local skill file, read it yourself and do not delegate it to a task.\n\n## Available Skills\n\nThe following skills provide specialized instructions for specific tasks. When a task matches a skill description, activate that skill before proceeding so its full instructions are loaded. Skill instructions and supporting resources stay lazy until activation or explicit file reads.\n\n- **e2e-flue-skill** — Deterministic skill used by the Braintrust Flue instrumentation e2e test.\n\nDate: Mon, Aug 3, 2026\nWorking directory: .\n\nDirectory structure:\n.agents\n.flue\n.flue-build\n__cassettes__\n__snapshots__\nassertions.ts\ncassette-filter.mjs\nconstants.mjs\ndist\nnode_modules\npackage.json\npnpm-lock.yaml\nscenario.cli.mjs\nscenario.impl.mjs\nscenario.mjs\nscenario.test.ts\nscenario.ts\nscenario.v2.ts\nv2-assertions.ts\nversions", - "role": "developer" - }, - { - "content": [ - { - "text": "Run the skill named \"e2e-flue-skill\".\n\nArguments:\n{\n \"marker\": \"SKILL_DONE\"\n}", - "type": "input_text" - } - ], - "role": "user" - }, - { - "arguments": "{\"description\":\"Run the deterministic Flue instrumentation e2e test skill e2e-flue-skill.\",\"prompt\":\"Run the skill named \\\"e2e-flue-skill\\\" with arguments {\\\"marker\\\":\\\"SKILL_DONE\\\"}. Output only the marker and no extra text when finished.\",\"cwd\":\".\"}", - "call_id": "call_R2a5LOml0NZqq5XTcTCKfbMW", - "id": "fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb", - "name": "task", - "type": "function_call" - }, - { - "call_id": "call_R2a5LOml0NZqq5XTcTCKfbMW", - "output": "SKILL_DONE", - "type": "function_call_output" - } - ], - "model": "gpt-5.4-nano", - "prompt_cache_key": "[REDACTED]", - "reasoning": { - "effort": "none" - }, - "store": false, - "stream": true, - "tools": [ - { - "description": "Read a file or list a directory. For files, output is truncated to 2000 lines or 50KB — use offset/limit for large files. For directories, returns the list of entries.", - "name": "read", - "parameters": { - "properties": { - "limit": { - "description": "Maximum number of lines to read", - "type": "number" - }, - "offset": { - "description": "Line number to start from (1-indexed)", - "type": "number" - }, - "path": { - "description": "Path to the file to read", - "type": "string" - } - }, - "required": ["path"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Write content to a file. Creates the file and parent directories if they do not exist.", - "name": "write", - "parameters": { - "properties": { - "content": { - "description": "Content to write to the file", - "type": "string" - }, - "path": { - "description": "Path to the file to write", - "type": "string" - } - }, - "required": ["path", "content"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Edit a file using exact text replacement. The oldText must match a unique region of the file. Use replaceAll to replace all occurrences.", - "name": "edit", - "parameters": { - "properties": { - "newText": { - "description": "Replacement text", - "type": "string" - }, - "oldText": { - "description": "Exact text to find (must be unique)", - "type": "string" - }, - "path": { - "description": "Path to the file to edit", - "type": "string" - }, - "replaceAll": { - "description": "Replace all occurrences", - "type": "boolean" - } - }, - "required": ["path", "oldText", "newText"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Execute a bash command. Returns stdout and stderr. Output is truncated to the last 2000 lines or 50KB.", - "name": "bash", - "parameters": { - "properties": { - "command": { - "description": "Bash command to execute", - "type": "string" - }, - "timeout": { - "description": "Timeout in seconds", - "type": "number" - } - }, - "required": ["command"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Search file contents for a regex pattern. Returns matching lines with file paths and line numbers.", - "name": "grep", - "parameters": { - "properties": { - "include": { - "description": "Glob filter, e.g. \"*.ts\"", - "type": "string" - }, - "path": { - "description": "Directory or file to search (default: .)", - "type": "string" - }, - "pattern": { - "description": "Search pattern (regex)", - "type": "string" - } - }, - "required": ["pattern"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Find files by filename pattern using shell find -name semantics. Returns matching file paths.", - "name": "glob", - "parameters": { - "properties": { - "path": { - "description": "Directory to search in (default: .)", - "type": "string" - }, - "pattern": { - "description": "Filename pattern, e.g. \"*.ts\"", - "type": "string" - } - }, - "required": ["pattern"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Delegate a focused task to a detached child agent with its own context. Use this for independent research, file exploration, or parallel work. The task returns only its final answer to this conversation. No subagents are currently defined.", - "name": "task", - "parameters": { - "properties": { - "agent": { - "description": "Declared subagent to use for the child agent", - "type": "string" - }, - "cwd": { - "description": "Working directory for the child agent. AGENTS.md and skills are discovered from here.", - "type": "string" - }, - "description": { - "description": "Short human-readable label for the delegated work", - "type": "string" - }, - "prompt": { - "description": "Focused instructions for the child agent", - "type": "string" - } - }, - "required": ["prompt"], - "type": "object" - }, - "strict": false, - "type": "function" - } - ] - } - }, - "headers": {}, - "method": "POST", - "url": "https://api.openai.com/v1/responses" - }, - "response": { - "body": { - "chunks": [ - "event: response.created\ndata: {\"type\":\"response.created\",\"response\":{\"id\":\"resp_0f099b3fce0721fd016a705b70d65881a3b5bc873ea54e7c00\",\"object\":\"response\",\"created_at\":1785748336,\"status\":\"in_progress\",\"background\":false,\"completed_at\":null,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":null,\"max_tool_calls\":null,\"model\":\"gpt-5.4-nano-2026-03-17\",\"moderation\":null,\"output\":[],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":\"workflow:instrumentation:01KZ3E9TWTAEK4P4P3F63HKAKC::default::sk\",\"prompt_cache_retention\":\"24h\",\"reasoning\":{\"context\":\"current_turn\",\"effort\":\"none\",\"mode\":\"standard\",\"summary\":null},\"safety_identifier\":null,\"service_tier\":\"auto\",\"store\":false,\"temperature\":1,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tool_usage\":{\"image_gen\":{\"input_tokens\":0,\"input_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"output_tokens\":0,\"output_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"total_tokens\":0},\"web_search\":{\"num_requests\":0}},\"tools\":[{\"type\":\"function\",\"description\":\"Read a file or list a directory. For files, output is truncated to 2000 lines or 50KB — use offset/limit for large files. For directories, returns the list of entries.\",\"name\":\"read\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to read\"},\"offset\":{\"type\":\"number\",\"description\":\"Line number to start from (1-indexed)\"},\"limit\":{\"type\":\"number\",\"description\":\"Maximum number of lines to read\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Write content to a file. Creates the file and parent directories if they do not exist.\",\"name\":\"write\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"content\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to write\"},\"content\":{\"type\":\"string\",\"description\":\"Content to write to the file\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Edit a file using exact text replacement. The oldText must match a unique region of the file. Use replaceAll to replace all occurrences.\",\"name\":\"edit\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"oldText\",\"newText\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to edit\"},\"oldText\":{\"type\":\"string\",\"description\":\"Exact text to find (must be unique)\"},\"newText\":{\"type\":\"string\",\"description\":\"Replacement text\"},\"replaceAll\":{\"type\":\"boolean\",\"description\":\"Replace all occurrences\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Execute a bash command. Returns stdout and stderr. Output is truncated to the last 2000 lines or 50KB.\",\"name\":\"bash\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"command\"],\"properties\":{\"command\":{\"type\":\"string\",\"description\":\"Bash command to execute\"},\"timeout\":{\"type\":\"number\",\"description\":\"Timeout in seconds\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Search file contents for a regex pattern. Returns matching lines with file paths and line numbers.\",\"name\":\"grep\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Search pattern (regex)\"},\"path\":{\"type\":\"string\",\"description\":\"Directory or file to search (default: .)\"},\"include\":{\"type\":\"string\",\"description\":\"Glob filter, e.g. \\\"*.ts\\\"\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Find files by filename pattern using shell find -name semantics. Returns matching file paths.\",\"name\":\"glob\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Filename pattern, e.g. \\\"*.ts\\\"\"},\"path\":{\"type\":\"string\",\"description\":\"Directory to search in (default: .)\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Delegate a focused task to a detached child agent with its own context. Use this for independent research, file exploration, or parallel work. The task returns only its final answer to this conversation. No subagents are currently defined.\",\"name\":\"task\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"prompt\"],\"properties\":{\"description\":{\"type\":\"string\",\"description\":\"Short human-readable label for the delegated work\"},\"prompt\":{\"type\":\"string\",\"description\":\"Focused instructions for the child agent\"},\"agent\":{\"type\":\"string\",\"description\":\"Declared subagent to use for the child agent\"},\"cwd\":{\"type\":\"string\",\"description\":\"Working directory for the child agent. AGENTS.md and skills are discovered from here.\"}}},\"strict\":false}],\"top_logprobs\":0,\"top_p\":0.98,\"truncation\":\"disabled\",\"usage\":null,\"user\":null,\"metadata\":{}},\"sequence_number\":0}", - "event: response.in_progress\ndata: {\"type\":\"response.in_progress\",\"response\":{\"id\":\"resp_0f099b3fce0721fd016a705b70d65881a3b5bc873ea54e7c00\",\"object\":\"response\",\"created_at\":1785748336,\"status\":\"in_progress\",\"background\":false,\"completed_at\":null,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":null,\"max_tool_calls\":null,\"model\":\"gpt-5.4-nano-2026-03-17\",\"moderation\":null,\"output\":[],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":\"workflow:instrumentation:01KZ3E9TWTAEK4P4P3F63HKAKC::default::sk\",\"prompt_cache_retention\":\"24h\",\"reasoning\":{\"context\":\"current_turn\",\"effort\":\"none\",\"mode\":\"standard\",\"summary\":null},\"safety_identifier\":null,\"service_tier\":\"auto\",\"store\":false,\"temperature\":1,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tool_usage\":{\"image_gen\":{\"input_tokens\":0,\"input_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"output_tokens\":0,\"output_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"total_tokens\":0},\"web_search\":{\"num_requests\":0}},\"tools\":[{\"type\":\"function\",\"description\":\"Read a file or list a directory. For files, output is truncated to 2000 lines or 50KB — use offset/limit for large files. For directories, returns the list of entries.\",\"name\":\"read\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to read\"},\"offset\":{\"type\":\"number\",\"description\":\"Line number to start from (1-indexed)\"},\"limit\":{\"type\":\"number\",\"description\":\"Maximum number of lines to read\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Write content to a file. Creates the file and parent directories if they do not exist.\",\"name\":\"write\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"content\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to write\"},\"content\":{\"type\":\"string\",\"description\":\"Content to write to the file\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Edit a file using exact text replacement. The oldText must match a unique region of the file. Use replaceAll to replace all occurrences.\",\"name\":\"edit\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"oldText\",\"newText\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to edit\"},\"oldText\":{\"type\":\"string\",\"description\":\"Exact text to find (must be unique)\"},\"newText\":{\"type\":\"string\",\"description\":\"Replacement text\"},\"replaceAll\":{\"type\":\"boolean\",\"description\":\"Replace all occurrences\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Execute a bash command. Returns stdout and stderr. Output is truncated to the last 2000 lines or 50KB.\",\"name\":\"bash\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"command\"],\"properties\":{\"command\":{\"type\":\"string\",\"description\":\"Bash command to execute\"},\"timeout\":{\"type\":\"number\",\"description\":\"Timeout in seconds\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Search file contents for a regex pattern. Returns matching lines with file paths and line numbers.\",\"name\":\"grep\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Search pattern (regex)\"},\"path\":{\"type\":\"string\",\"description\":\"Directory or file to search (default: .)\"},\"include\":{\"type\":\"string\",\"description\":\"Glob filter, e.g. \\\"*.ts\\\"\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Find files by filename pattern using shell find -name semantics. Returns matching file paths.\",\"name\":\"glob\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Filename pattern, e.g. \\\"*.ts\\\"\"},\"path\":{\"type\":\"string\",\"description\":\"Directory to search in (default: .)\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Delegate a focused task to a detached child agent with its own context. Use this for independent research, file exploration, or parallel work. The task returns only its final answer to this conversation. No subagents are currently defined.\",\"name\":\"task\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"prompt\"],\"properties\":{\"description\":{\"type\":\"string\",\"description\":\"Short human-readable label for the delegated work\"},\"prompt\":{\"type\":\"string\",\"description\":\"Focused instructions for the child agent\"},\"agent\":{\"type\":\"string\",\"description\":\"Declared subagent to use for the child agent\"},\"cwd\":{\"type\":\"string\",\"description\":\"Working directory for the child agent. AGENTS.md and skills are discovered from here.\"}}},\"strict\":false}],\"top_logprobs\":0,\"top_p\":0.98,\"truncation\":\"disabled\",\"usage\":null,\"user\":null,\"metadata\":{}},\"sequence_number\":1}", - "event: response.output_item.added\ndata: {\"type\":\"response.output_item.added\",\"item\":{\"id\":\"msg_0f099b3fce0721fd016a705b71558881a38c53a767e09d9e31\",\"type\":\"message\",\"status\":\"in_progress\",\"content\":[],\"phase\":\"final_answer\",\"role\":\"assistant\"},\"output_index\":0,\"sequence_number\":2}", - "event: response.content_part.added\ndata: {\"type\":\"response.content_part.added\",\"content_index\":0,\"item_id\":\"msg_0f099b3fce0721fd016a705b71558881a38c53a767e09d9e31\",\"output_index\":0,\"part\":{\"type\":\"output_text\",\"annotations\":[],\"logprobs\":[],\"text\":\"\"},\"sequence_number\":3}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"SK\",\"item_id\":\"msg_0f099b3fce0721fd016a705b71558881a38c53a767e09d9e31\",\"logprobs\":[],\"obfuscation\":\"iSZ7rWanncJQ68\",\"output_index\":0,\"sequence_number\":4}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"ILL\",\"item_id\":\"msg_0f099b3fce0721fd016a705b71558881a38c53a767e09d9e31\",\"logprobs\":[],\"obfuscation\":\"PQly4lHaqRXSu\",\"output_index\":0,\"sequence_number\":5}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"_DONE\",\"item_id\":\"msg_0f099b3fce0721fd016a705b71558881a38c53a767e09d9e31\",\"logprobs\":[],\"obfuscation\":\"WSIPPwqqidY\",\"output_index\":0,\"sequence_number\":6}", - "event: response.output_text.done\ndata: {\"type\":\"response.output_text.done\",\"content_index\":0,\"item_id\":\"msg_0f099b3fce0721fd016a705b71558881a38c53a767e09d9e31\",\"logprobs\":[],\"output_index\":0,\"sequence_number\":7,\"text\":\"SKILL_DONE\"}", - "event: response.content_part.done\ndata: {\"type\":\"response.content_part.done\",\"content_index\":0,\"item_id\":\"msg_0f099b3fce0721fd016a705b71558881a38c53a767e09d9e31\",\"output_index\":0,\"part\":{\"type\":\"output_text\",\"annotations\":[],\"logprobs\":[],\"text\":\"SKILL_DONE\"},\"sequence_number\":8}", - "event: response.output_item.done\ndata: {\"type\":\"response.output_item.done\",\"item\":{\"id\":\"msg_0f099b3fce0721fd016a705b71558881a38c53a767e09d9e31\",\"type\":\"message\",\"status\":\"completed\",\"content\":[{\"type\":\"output_text\",\"annotations\":[],\"logprobs\":[],\"text\":\"SKILL_DONE\"}],\"phase\":\"final_answer\",\"role\":\"assistant\"},\"output_index\":0,\"sequence_number\":9}", - "event: response.completed\ndata: {\"type\":\"response.completed\",\"response\":{\"id\":\"resp_0f099b3fce0721fd016a705b70d65881a3b5bc873ea54e7c00\",\"object\":\"response\",\"created_at\":1785748336,\"status\":\"completed\",\"background\":false,\"completed_at\":1785748337,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":null,\"max_tool_calls\":null,\"model\":\"gpt-5.4-nano-2026-03-17\",\"moderation\":null,\"output\":[{\"id\":\"msg_0f099b3fce0721fd016a705b71558881a38c53a767e09d9e31\",\"type\":\"message\",\"status\":\"completed\",\"content\":[{\"type\":\"output_text\",\"annotations\":[],\"logprobs\":[],\"text\":\"SKILL_DONE\"}],\"phase\":\"final_answer\",\"role\":\"assistant\"}],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":\"workflow:instrumentation:01KZ3E9TWTAEK4P4P3F63HKAKC::default::sk\",\"prompt_cache_retention\":\"24h\",\"reasoning\":{\"context\":\"current_turn\",\"effort\":\"none\",\"mode\":\"standard\",\"summary\":null},\"safety_identifier\":null,\"service_tier\":\"default\",\"store\":false,\"temperature\":1,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tool_usage\":{\"image_gen\":{\"input_tokens\":0,\"input_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"output_tokens\":0,\"output_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"total_tokens\":0},\"web_search\":{\"num_requests\":0}},\"tools\":[{\"type\":\"function\",\"description\":\"Read a file or list a directory. For files, output is truncated to 2000 lines or 50KB — use offset/limit for large files. For directories, returns the list of entries.\",\"name\":\"read\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to read\"},\"offset\":{\"type\":\"number\",\"description\":\"Line number to start from (1-indexed)\"},\"limit\":{\"type\":\"number\",\"description\":\"Maximum number of lines to read\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Write content to a file. Creates the file and parent directories if they do not exist.\",\"name\":\"write\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"content\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to write\"},\"content\":{\"type\":\"string\",\"description\":\"Content to write to the file\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Edit a file using exact text replacement. The oldText must match a unique region of the file. Use replaceAll to replace all occurrences.\",\"name\":\"edit\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"oldText\",\"newText\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to edit\"},\"oldText\":{\"type\":\"string\",\"description\":\"Exact text to find (must be unique)\"},\"newText\":{\"type\":\"string\",\"description\":\"Replacement text\"},\"replaceAll\":{\"type\":\"boolean\",\"description\":\"Replace all occurrences\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Execute a bash command. Returns stdout and stderr. Output is truncated to the last 2000 lines or 50KB.\",\"name\":\"bash\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"command\"],\"properties\":{\"command\":{\"type\":\"string\",\"description\":\"Bash command to execute\"},\"timeout\":{\"type\":\"number\",\"description\":\"Timeout in seconds\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Search file contents for a regex pattern. Returns matching lines with file paths and line numbers.\",\"name\":\"grep\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Search pattern (regex)\"},\"path\":{\"type\":\"string\",\"description\":\"Directory or file to search (default: .)\"},\"include\":{\"type\":\"string\",\"description\":\"Glob filter, e.g. \\\"*.ts\\\"\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Find files by filename pattern using shell find -name semantics. Returns matching file paths.\",\"name\":\"glob\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Filename pattern, e.g. \\\"*.ts\\\"\"},\"path\":{\"type\":\"string\",\"description\":\"Directory to search in (default: .)\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Delegate a focused task to a detached child agent with its own context. Use this for independent research, file exploration, or parallel work. The task returns only its final answer to this conversation. No subagents are currently defined.\",\"name\":\"task\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"prompt\"],\"properties\":{\"description\":{\"type\":\"string\",\"description\":\"Short human-readable label for the delegated work\"},\"prompt\":{\"type\":\"string\",\"description\":\"Focused instructions for the child agent\"},\"agent\":{\"type\":\"string\",\"description\":\"Declared subagent to use for the child agent\"},\"cwd\":{\"type\":\"string\",\"description\":\"Working directory for the child agent. AGENTS.md and skills are discovered from here.\"}}},\"strict\":false}],\"top_logprobs\":0,\"top_p\":0.98,\"truncation\":\"disabled\",\"usage\":{\"input_tokens\":969,\"input_tokens_details\":{\"cache_write_tokens\":0,\"cached_tokens\":0},\"output_tokens\":7,\"output_tokens_details\":{\"reasoning_tokens\":0},\"total_tokens\":976},\"user\":null,\"metadata\":{}},\"sequence_number\":10}" - ], - "kind": "sse" - }, - "headers": { - "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", - "alt-svc": "h3=\":443\"; ma=86400", - "cf-cache-status": "DYNAMIC", - "cf-ray": "a2543320eec07bb6-IAD", - "connection": "keep-alive", - "content-type": "text/event-stream; charset=utf-8", - "date": "Mon, 03 Aug 2026 09:12:17 GMT", - "openai-organization": "braintrust-data", - "openai-processing-ms": "226", - "openai-project": "proj_vsCSXafhhByzWOThMrJcZiw9", - "openai-version": "2020-10-01", - "server": "cloudflare", - "set-cookie": "[REDACTED]", - "strict-transport-security": "max-age=31536000; includeSubDomains; preload", - "transfer-encoding": "chunked", - "x-content-type-options": "nosniff", - "x-ratelimit-limit-requests": "30000", - "x-ratelimit-limit-tokens": "180000000", - "x-ratelimit-remaining-requests": "29999", - "x-ratelimit-remaining-tokens": "179998575", - "x-ratelimit-reset-requests": "2ms", - "x-ratelimit-reset-tokens": "0s", - "x-request-id": "req_0f63631954824ce89d221047a6871115" - }, - "status": 200, - "statusText": "OK" - } - }, - { - "callIndex": 16, - "id": "eea2c2f028018d92", - "matchKey": "POST api.openai.com/v1/responses", - "recordedAt": "2026-08-03T09:12:18.781Z", - "request": { - "body": { - "kind": "json", - "value": { - "input": [ - { - "content": "You are running in headless mode with no human operator. Work autonomously — never ask questions, never wait for user input. Make your best judgment and proceed independently.\n\nYou are a deterministic Flue instrumentation test agent. Follow user instructions exactly. When asked for a marker, output only that marker and no extra text. When running a local skill file, read it yourself and do not delegate it to a task.\n\n## Available Skills\n\nThe following skills provide specialized instructions for specific tasks. When a task matches a skill description, activate that skill before proceeding so its full instructions are loaded. Skill instructions and supporting resources stay lazy until activation or explicit file reads.\n\n- **e2e-flue-skill** — Deterministic skill used by the Braintrust Flue instrumentation e2e test.\n\nDate: Mon, Aug 3, 2026\nWorking directory: .\n\nDirectory structure:\n.agents\n.flue\n.flue-build\n__cassettes__\n__snapshots__\nassertions.ts\ncassette-filter.mjs\nconstants.mjs\ndist\nnode_modules\npackage.json\npnpm-lock.yaml\nscenario.cli.mjs\nscenario.impl.mjs\nscenario.mjs\nscenario.test.ts\nscenario.ts\nscenario.v2.ts\nv2-assertions.ts\nversions", - "role": "system" - }, - { - "content": [ - { - "text": "Reply with exactly TASK_DONE and no other text.", - "type": "input_text" - } - ], - "role": "user" - } - ], - "model": "gpt-4o-mini", - "prompt_cache_key": "[REDACTED]", - "store": false, - "stream": true, - "tools": [ - { - "description": "Read a file or list a directory. For files, output is truncated to 2000 lines or 50KB — use offset/limit for large files. For directories, returns the list of entries.", - "name": "read", - "parameters": { - "properties": { - "limit": { - "description": "Maximum number of lines to read", - "type": "number" - }, - "offset": { - "description": "Line number to start from (1-indexed)", - "type": "number" - }, - "path": { - "description": "Path to the file to read", - "type": "string" - } - }, - "required": ["path"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Write content to a file. Creates the file and parent directories if they do not exist.", - "name": "write", - "parameters": { - "properties": { - "content": { - "description": "Content to write to the file", - "type": "string" - }, - "path": { - "description": "Path to the file to write", - "type": "string" - } - }, - "required": ["path", "content"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Edit a file using exact text replacement. The oldText must match a unique region of the file. Use replaceAll to replace all occurrences.", - "name": "edit", - "parameters": { - "properties": { - "newText": { - "description": "Replacement text", - "type": "string" - }, - "oldText": { - "description": "Exact text to find (must be unique)", - "type": "string" - }, - "path": { - "description": "Path to the file to edit", - "type": "string" - }, - "replaceAll": { - "description": "Replace all occurrences", - "type": "boolean" - } - }, - "required": ["path", "oldText", "newText"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Execute a bash command. Returns stdout and stderr. Output is truncated to the last 2000 lines or 50KB.", - "name": "bash", - "parameters": { - "properties": { - "command": { - "description": "Bash command to execute", - "type": "string" - }, - "timeout": { - "description": "Timeout in seconds", - "type": "number" - } - }, - "required": ["command"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Search file contents for a regex pattern. Returns matching lines with file paths and line numbers.", - "name": "grep", - "parameters": { - "properties": { - "include": { - "description": "Glob filter, e.g. \"*.ts\"", - "type": "string" - }, - "path": { - "description": "Directory or file to search (default: .)", - "type": "string" - }, - "pattern": { - "description": "Search pattern (regex)", - "type": "string" - } - }, - "required": ["pattern"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Find files by filename pattern using shell find -name semantics. Returns matching file paths.", - "name": "glob", - "parameters": { - "properties": { - "path": { - "description": "Directory to search in (default: .)", - "type": "string" - }, - "pattern": { - "description": "Filename pattern, e.g. \"*.ts\"", - "type": "string" - } - }, - "required": ["pattern"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Delegate a focused task to a detached child agent with its own context. Use this for independent research, file exploration, or parallel work. The task returns only its final answer to this conversation. No subagents are currently defined.", - "name": "task", - "parameters": { - "properties": { - "agent": { - "description": "Declared subagent to use for the child agent", - "type": "string" - }, - "cwd": { - "description": "Working directory for the child agent. AGENTS.md and skills are discovered from here.", - "type": "string" - }, - "description": { - "description": "Short human-readable label for the delegated work", - "type": "string" - }, - "prompt": { - "description": "Focused instructions for the child agent", - "type": "string" - } - }, - "required": ["prompt"], - "type": "object" - }, - "strict": false, - "type": "function" - } - ] - } - }, - "headers": {}, - "method": "POST", - "url": "https://api.openai.com/v1/responses" - }, - "response": { - "body": { - "chunks": [ - "event: response.created\ndata: {\"type\":\"response.created\",\"response\":{\"id\":\"resp_0f3adfdb1510c165016a705b7192bc81a094e1dbada8d8b42b\",\"object\":\"response\",\"created_at\":1785748337,\"status\":\"in_progress\",\"background\":false,\"completed_at\":null,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":null,\"max_tool_calls\":null,\"model\":\"gpt-4o-mini-2024-07-18\",\"moderation\":null,\"output\":[],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":\"workflow:instrumentation:01KZ3E9TWTAEK4P4P3F63HKAKC::default::ta\",\"prompt_cache_retention\":\"in_memory\",\"reasoning\":{\"context\":null,\"effort\":null,\"summary\":null},\"safety_identifier\":null,\"service_tier\":\"auto\",\"store\":false,\"temperature\":1,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tool_usage\":{\"image_gen\":{\"input_tokens\":0,\"input_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"output_tokens\":0,\"output_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"total_tokens\":0},\"web_search\":{\"num_requests\":0}},\"tools\":[{\"type\":\"function\",\"description\":\"Read a file or list a directory. For files, output is truncated to 2000 lines or 50KB — use offset/limit for large files. For directories, returns the list of entries.\",\"name\":\"read\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to read\"},\"offset\":{\"type\":\"number\",\"description\":\"Line number to start from (1-indexed)\"},\"limit\":{\"type\":\"number\",\"description\":\"Maximum number of lines to read\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Write content to a file. Creates the file and parent directories if they do not exist.\",\"name\":\"write\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"content\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to write\"},\"content\":{\"type\":\"string\",\"description\":\"Content to write to the file\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Edit a file using exact text replacement. The oldText must match a unique region of the file. Use replaceAll to replace all occurrences.\",\"name\":\"edit\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"oldText\",\"newText\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to edit\"},\"oldText\":{\"type\":\"string\",\"description\":\"Exact text to find (must be unique)\"},\"newText\":{\"type\":\"string\",\"description\":\"Replacement text\"},\"replaceAll\":{\"type\":\"boolean\",\"description\":\"Replace all occurrences\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Execute a bash command. Returns stdout and stderr. Output is truncated to the last 2000 lines or 50KB.\",\"name\":\"bash\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"command\"],\"properties\":{\"command\":{\"type\":\"string\",\"description\":\"Bash command to execute\"},\"timeout\":{\"type\":\"number\",\"description\":\"Timeout in seconds\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Search file contents for a regex pattern. Returns matching lines with file paths and line numbers.\",\"name\":\"grep\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Search pattern (regex)\"},\"path\":{\"type\":\"string\",\"description\":\"Directory or file to search (default: .)\"},\"include\":{\"type\":\"string\",\"description\":\"Glob filter, e.g. \\\"*.ts\\\"\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Find files by filename pattern using shell find -name semantics. Returns matching file paths.\",\"name\":\"glob\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Filename pattern, e.g. \\\"*.ts\\\"\"},\"path\":{\"type\":\"string\",\"description\":\"Directory to search in (default: .)\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Delegate a focused task to a detached child agent with its own context. Use this for independent research, file exploration, or parallel work. The task returns only its final answer to this conversation. No subagents are currently defined.\",\"name\":\"task\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"prompt\"],\"properties\":{\"description\":{\"type\":\"string\",\"description\":\"Short human-readable label for the delegated work\"},\"prompt\":{\"type\":\"string\",\"description\":\"Focused instructions for the child agent\"},\"agent\":{\"type\":\"string\",\"description\":\"Declared subagent to use for the child agent\"},\"cwd\":{\"type\":\"string\",\"description\":\"Working directory for the child agent. AGENTS.md and skills are discovered from here.\"}}},\"strict\":false}],\"top_logprobs\":0,\"top_p\":1,\"truncation\":\"disabled\",\"usage\":null,\"user\":null,\"metadata\":{}},\"sequence_number\":0}", - "event: response.in_progress\ndata: {\"type\":\"response.in_progress\",\"response\":{\"id\":\"resp_0f3adfdb1510c165016a705b7192bc81a094e1dbada8d8b42b\",\"object\":\"response\",\"created_at\":1785748337,\"status\":\"in_progress\",\"background\":false,\"completed_at\":null,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":null,\"max_tool_calls\":null,\"model\":\"gpt-4o-mini-2024-07-18\",\"moderation\":null,\"output\":[],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":\"workflow:instrumentation:01KZ3E9TWTAEK4P4P3F63HKAKC::default::ta\",\"prompt_cache_retention\":\"in_memory\",\"reasoning\":{\"context\":null,\"effort\":null,\"summary\":null},\"safety_identifier\":null,\"service_tier\":\"auto\",\"store\":false,\"temperature\":1,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tool_usage\":{\"image_gen\":{\"input_tokens\":0,\"input_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"output_tokens\":0,\"output_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"total_tokens\":0},\"web_search\":{\"num_requests\":0}},\"tools\":[{\"type\":\"function\",\"description\":\"Read a file or list a directory. For files, output is truncated to 2000 lines or 50KB — use offset/limit for large files. For directories, returns the list of entries.\",\"name\":\"read\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to read\"},\"offset\":{\"type\":\"number\",\"description\":\"Line number to start from (1-indexed)\"},\"limit\":{\"type\":\"number\",\"description\":\"Maximum number of lines to read\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Write content to a file. Creates the file and parent directories if they do not exist.\",\"name\":\"write\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"content\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to write\"},\"content\":{\"type\":\"string\",\"description\":\"Content to write to the file\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Edit a file using exact text replacement. The oldText must match a unique region of the file. Use replaceAll to replace all occurrences.\",\"name\":\"edit\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"oldText\",\"newText\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to edit\"},\"oldText\":{\"type\":\"string\",\"description\":\"Exact text to find (must be unique)\"},\"newText\":{\"type\":\"string\",\"description\":\"Replacement text\"},\"replaceAll\":{\"type\":\"boolean\",\"description\":\"Replace all occurrences\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Execute a bash command. Returns stdout and stderr. Output is truncated to the last 2000 lines or 50KB.\",\"name\":\"bash\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"command\"],\"properties\":{\"command\":{\"type\":\"string\",\"description\":\"Bash command to execute\"},\"timeout\":{\"type\":\"number\",\"description\":\"Timeout in seconds\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Search file contents for a regex pattern. Returns matching lines with file paths and line numbers.\",\"name\":\"grep\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Search pattern (regex)\"},\"path\":{\"type\":\"string\",\"description\":\"Directory or file to search (default: .)\"},\"include\":{\"type\":\"string\",\"description\":\"Glob filter, e.g. \\\"*.ts\\\"\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Find files by filename pattern using shell find -name semantics. Returns matching file paths.\",\"name\":\"glob\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Filename pattern, e.g. \\\"*.ts\\\"\"},\"path\":{\"type\":\"string\",\"description\":\"Directory to search in (default: .)\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Delegate a focused task to a detached child agent with its own context. Use this for independent research, file exploration, or parallel work. The task returns only its final answer to this conversation. No subagents are currently defined.\",\"name\":\"task\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"prompt\"],\"properties\":{\"description\":{\"type\":\"string\",\"description\":\"Short human-readable label for the delegated work\"},\"prompt\":{\"type\":\"string\",\"description\":\"Focused instructions for the child agent\"},\"agent\":{\"type\":\"string\",\"description\":\"Declared subagent to use for the child agent\"},\"cwd\":{\"type\":\"string\",\"description\":\"Working directory for the child agent. AGENTS.md and skills are discovered from here.\"}}},\"strict\":false}],\"top_logprobs\":0,\"top_p\":1,\"truncation\":\"disabled\",\"usage\":null,\"user\":null,\"metadata\":{}},\"sequence_number\":1}", - "event: response.output_item.added\ndata: {\"type\":\"response.output_item.added\",\"item\":{\"id\":\"msg_0f3adfdb1510c165016a705b72843c81a0b30d242da683a882\",\"type\":\"message\",\"status\":\"in_progress\",\"content\":[],\"role\":\"assistant\"},\"output_index\":0,\"sequence_number\":2}", - "event: response.content_part.added\ndata: {\"type\":\"response.content_part.added\",\"content_index\":0,\"item_id\":\"msg_0f3adfdb1510c165016a705b72843c81a0b30d242da683a882\",\"output_index\":0,\"part\":{\"type\":\"output_text\",\"annotations\":[],\"logprobs\":[],\"text\":\"\"},\"sequence_number\":3}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"TASK\",\"item_id\":\"msg_0f3adfdb1510c165016a705b72843c81a0b30d242da683a882\",\"logprobs\":[],\"obfuscation\":\"mlb5p2sBa8Nm\",\"output_index\":0,\"sequence_number\":4}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"_DONE\",\"item_id\":\"msg_0f3adfdb1510c165016a705b72843c81a0b30d242da683a882\",\"logprobs\":[],\"obfuscation\":\"oWnwIAUMpIA\",\"output_index\":0,\"sequence_number\":5}", - "event: response.output_text.done\ndata: {\"type\":\"response.output_text.done\",\"content_index\":0,\"item_id\":\"msg_0f3adfdb1510c165016a705b72843c81a0b30d242da683a882\",\"logprobs\":[],\"output_index\":0,\"sequence_number\":6,\"text\":\"TASK_DONE\"}", - "event: response.content_part.done\ndata: {\"type\":\"response.content_part.done\",\"content_index\":0,\"item_id\":\"msg_0f3adfdb1510c165016a705b72843c81a0b30d242da683a882\",\"output_index\":0,\"part\":{\"type\":\"output_text\",\"annotations\":[],\"logprobs\":[],\"text\":\"TASK_DONE\"},\"sequence_number\":7}", - "event: response.output_item.done\ndata: {\"type\":\"response.output_item.done\",\"item\":{\"id\":\"msg_0f3adfdb1510c165016a705b72843c81a0b30d242da683a882\",\"type\":\"message\",\"status\":\"completed\",\"content\":[{\"type\":\"output_text\",\"annotations\":[],\"logprobs\":[],\"text\":\"TASK_DONE\"}],\"role\":\"assistant\"},\"output_index\":0,\"sequence_number\":8}", - "event: response.completed\ndata: {\"type\":\"response.completed\",\"response\":{\"id\":\"resp_0f3adfdb1510c165016a705b7192bc81a094e1dbada8d8b42b\",\"object\":\"response\",\"created_at\":1785748337,\"status\":\"completed\",\"background\":false,\"completed_at\":1785748338,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":null,\"max_tool_calls\":null,\"model\":\"gpt-4o-mini-2024-07-18\",\"moderation\":null,\"output\":[{\"id\":\"msg_0f3adfdb1510c165016a705b72843c81a0b30d242da683a882\",\"type\":\"message\",\"status\":\"completed\",\"content\":[{\"type\":\"output_text\",\"annotations\":[],\"logprobs\":[],\"text\":\"TASK_DONE\"}],\"role\":\"assistant\"}],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":\"workflow:instrumentation:01KZ3E9TWTAEK4P4P3F63HKAKC::default::ta\",\"prompt_cache_retention\":\"in_memory\",\"reasoning\":{\"context\":null,\"effort\":null,\"summary\":null},\"safety_identifier\":null,\"service_tier\":\"default\",\"store\":false,\"temperature\":1,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tool_usage\":{\"image_gen\":{\"input_tokens\":0,\"input_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"output_tokens\":0,\"output_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"total_tokens\":0},\"web_search\":{\"num_requests\":0}},\"tools\":[{\"type\":\"function\",\"description\":\"Read a file or list a directory. For files, output is truncated to 2000 lines or 50KB — use offset/limit for large files. For directories, returns the list of entries.\",\"name\":\"read\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to read\"},\"offset\":{\"type\":\"number\",\"description\":\"Line number to start from (1-indexed)\"},\"limit\":{\"type\":\"number\",\"description\":\"Maximum number of lines to read\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Write content to a file. Creates the file and parent directories if they do not exist.\",\"name\":\"write\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"content\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to write\"},\"content\":{\"type\":\"string\",\"description\":\"Content to write to the file\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Edit a file using exact text replacement. The oldText must match a unique region of the file. Use replaceAll to replace all occurrences.\",\"name\":\"edit\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"oldText\",\"newText\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to edit\"},\"oldText\":{\"type\":\"string\",\"description\":\"Exact text to find (must be unique)\"},\"newText\":{\"type\":\"string\",\"description\":\"Replacement text\"},\"replaceAll\":{\"type\":\"boolean\",\"description\":\"Replace all occurrences\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Execute a bash command. Returns stdout and stderr. Output is truncated to the last 2000 lines or 50KB.\",\"name\":\"bash\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"command\"],\"properties\":{\"command\":{\"type\":\"string\",\"description\":\"Bash command to execute\"},\"timeout\":{\"type\":\"number\",\"description\":\"Timeout in seconds\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Search file contents for a regex pattern. Returns matching lines with file paths and line numbers.\",\"name\":\"grep\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Search pattern (regex)\"},\"path\":{\"type\":\"string\",\"description\":\"Directory or file to search (default: .)\"},\"include\":{\"type\":\"string\",\"description\":\"Glob filter, e.g. \\\"*.ts\\\"\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Find files by filename pattern using shell find -name semantics. Returns matching file paths.\",\"name\":\"glob\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Filename pattern, e.g. \\\"*.ts\\\"\"},\"path\":{\"type\":\"string\",\"description\":\"Directory to search in (default: .)\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Delegate a focused task to a detached child agent with its own context. Use this for independent research, file exploration, or parallel work. The task returns only its final answer to this conversation. No subagents are currently defined.\",\"name\":\"task\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"prompt\"],\"properties\":{\"description\":{\"type\":\"string\",\"description\":\"Short human-readable label for the delegated work\"},\"prompt\":{\"type\":\"string\",\"description\":\"Focused instructions for the child agent\"},\"agent\":{\"type\":\"string\",\"description\":\"Declared subagent to use for the child agent\"},\"cwd\":{\"type\":\"string\",\"description\":\"Working directory for the child agent. AGENTS.md and skills are discovered from here.\"}}},\"strict\":false}],\"top_logprobs\":0,\"top_p\":1,\"truncation\":\"disabled\",\"usage\":{\"input_tokens\":826,\"input_tokens_details\":{\"cache_write_tokens\":0,\"cached_tokens\":0},\"output_tokens\":4,\"output_tokens_details\":{\"reasoning_tokens\":0},\"total_tokens\":830},\"user\":null,\"metadata\":{}},\"sequence_number\":9}" - ], - "kind": "sse" - }, - "headers": { - "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", - "alt-svc": "h3=\":443\"; ma=86400", - "cf-cache-status": "DYNAMIC", - "cf-ray": "a254332588917bb6-IAD", - "connection": "keep-alive", - "content-type": "text/event-stream; charset=utf-8", - "date": "Mon, 03 Aug 2026 09:12:17 GMT", - "openai-organization": "braintrust-data", - "openai-processing-ms": "169", - "openai-project": "proj_vsCSXafhhByzWOThMrJcZiw9", - "openai-version": "2020-10-01", - "server": "cloudflare", - "set-cookie": "[REDACTED]", - "strict-transport-security": "max-age=31536000; includeSubDomains; preload", - "transfer-encoding": "chunked", - "x-content-type-options": "nosniff", - "x-request-id": "req_44ff6ef3357a4d78a0e4b836a0777717" - }, - "status": 200, - "statusText": "OK" - } - }, - { - "callIndex": 17, - "id": "dd8319160ba3e4cf", - "matchKey": "POST api.openai.com/v1/responses", - "recordedAt": "2026-08-03T09:12:19.721Z", - "request": { - "body": { - "kind": "json", - "value": { - "input": [ - { - "content": "You are a context summarization assistant. Your task is to read a conversation between a user and an AI coding assistant, then produce a structured summary following the exact format specified.\n\nDo NOT continue the conversation. Do NOT respond to any questions in the conversation. ONLY output the structured summary.", - "role": "system" - }, - { - "content": [ - { - "text": "\n[User]: Complete this instrumented research flow. Call exactly one tool per turn and wait for each tool result before choosing the next tool. Step 1: call lookup with query \"flue instrumentation\". Step 2: use the lookup result id as lookupId and call web_search with query \"Braintrust Flue reasoning stream instrumentation\". Step 3: use the first web_search result url and call summarize_source. After summarize_source returns, reply with exactly PROMPT_DONE and no other text.\n\n[Assistant tool calls]: lookup(query=\"flue instrumentation\")\n\n[Tool result]: {\"id\":\"flue-session-2026\",\"query\":\"flue instrumentation\",\"topic\":\"session instrumentation\"}\n\n[Assistant tool calls]: web_search(lookupId=\"flue-session-2026\", query=\"Braintrust Flue reasoning stream instrumentation\")\n\n[Tool result]: {\"lookupId\":\"flue-session-2026\",\"query\":\"Braintrust Flue reasoning stream instrumentation\",\"results\":[{\"title\":\"Flue reasoning stream instrumentation\",\"url\":\"https://example.test/flue/reasoning-streams\"}]}\n\n[Assistant tool calls]: summarize_source(url=\"https://example.test/flue/reasoning-streams\")\n\n[Tool result]: {\"summary\":\"Flue emits reasoning, tool execution, and LLM turn events separately.\",\"url\":\"https://example.test/flue/reasoning-streams\"}\n\n\nThis is the PREFIX of a turn that was too large to keep. The SUFFIX (recent work) is retained.\n\nSummarize the prefix to provide context for the retained suffix:\n\n## Original Request\n[What did the user ask for in this turn?]\n\n## Early Progress\n- [Key decisions and work done in the prefix]\n\n## Context for Suffix\n- [Information needed to understand the retained recent work]\n\nBe concise. Focus on what's needed to understand the kept suffix.", - "type": "input_text" - } - ], - "role": "user" - } - ], - "max_output_tokens": 32, - "model": "gpt-4o-mini", - "store": false, - "stream": true - } - }, - "headers": {}, - "method": "POST", - "url": "https://api.openai.com/v1/responses" - }, - "response": { - "body": { - "chunks": [ - "event: response.created\ndata: {\"type\":\"response.created\",\"response\":{\"id\":\"resp_04d28ba03840bb65016a705b72d36081a3a387e30f2739466c\",\"object\":\"response\",\"created_at\":1785748338,\"status\":\"in_progress\",\"background\":false,\"completed_at\":null,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":32,\"max_tool_calls\":null,\"model\":\"gpt-4o-mini-2024-07-18\",\"moderation\":null,\"output\":[],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":null,\"prompt_cache_retention\":\"in_memory\",\"reasoning\":{\"context\":null,\"effort\":null,\"summary\":null},\"safety_identifier\":null,\"service_tier\":\"auto\",\"store\":false,\"temperature\":1,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tool_usage\":{\"image_gen\":{\"input_tokens\":0,\"input_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"output_tokens\":0,\"output_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"total_tokens\":0},\"web_search\":{\"num_requests\":0}},\"tools\":[],\"top_logprobs\":0,\"top_p\":1,\"truncation\":\"disabled\",\"usage\":null,\"user\":null,\"metadata\":{}},\"sequence_number\":0}", - "event: response.in_progress\ndata: {\"type\":\"response.in_progress\",\"response\":{\"id\":\"resp_04d28ba03840bb65016a705b72d36081a3a387e30f2739466c\",\"object\":\"response\",\"created_at\":1785748338,\"status\":\"in_progress\",\"background\":false,\"completed_at\":null,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":32,\"max_tool_calls\":null,\"model\":\"gpt-4o-mini-2024-07-18\",\"moderation\":null,\"output\":[],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":null,\"prompt_cache_retention\":\"in_memory\",\"reasoning\":{\"context\":null,\"effort\":null,\"summary\":null},\"safety_identifier\":null,\"service_tier\":\"auto\",\"store\":false,\"temperature\":1,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tool_usage\":{\"image_gen\":{\"input_tokens\":0,\"input_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"output_tokens\":0,\"output_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"total_tokens\":0},\"web_search\":{\"num_requests\":0}},\"tools\":[],\"top_logprobs\":0,\"top_p\":1,\"truncation\":\"disabled\",\"usage\":null,\"user\":null,\"metadata\":{}},\"sequence_number\":1}", - "event: response.output_item.added\ndata: {\"type\":\"response.output_item.added\",\"item\":{\"id\":\"msg_04d28ba03840bb65016a705b7358a081a38e69464057840036\",\"type\":\"message\",\"status\":\"in_progress\",\"content\":[],\"role\":\"assistant\"},\"output_index\":0,\"sequence_number\":2}", - "event: response.content_part.added\ndata: {\"type\":\"response.content_part.added\",\"content_index\":0,\"item_id\":\"msg_04d28ba03840bb65016a705b7358a081a38e69464057840036\",\"output_index\":0,\"part\":{\"type\":\"output_text\",\"annotations\":[],\"logprobs\":[],\"text\":\"\"},\"sequence_number\":3}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"##\",\"item_id\":\"msg_04d28ba03840bb65016a705b7358a081a38e69464057840036\",\"logprobs\":[],\"obfuscation\":\"pnRDruuzK4zmSi\",\"output_index\":0,\"sequence_number\":4}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" Original\",\"item_id\":\"msg_04d28ba03840bb65016a705b7358a081a38e69464057840036\",\"logprobs\":[],\"obfuscation\":\"xeQeZBG\",\"output_index\":0,\"sequence_number\":5}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" Request\",\"item_id\":\"msg_04d28ba03840bb65016a705b7358a081a38e69464057840036\",\"logprobs\":[],\"obfuscation\":\"wDl6Xfa7\",\"output_index\":0,\"sequence_number\":6}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"\\n\",\"item_id\":\"msg_04d28ba03840bb65016a705b7358a081a38e69464057840036\",\"logprobs\":[],\"obfuscation\":\"SPfPR8TAgZd37wg\",\"output_index\":0,\"sequence_number\":7}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"The\",\"item_id\":\"msg_04d28ba03840bb65016a705b7358a081a38e69464057840036\",\"logprobs\":[],\"obfuscation\":\"cgHwyTaOZmFtZ\",\"output_index\":0,\"sequence_number\":8}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" user\",\"item_id\":\"msg_04d28ba03840bb65016a705b7358a081a38e69464057840036\",\"logprobs\":[],\"obfuscation\":\"1z0yUfwbdDc\",\"output_index\":0,\"sequence_number\":9}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" requested\",\"item_id\":\"msg_04d28ba03840bb65016a705b7358a081a38e69464057840036\",\"logprobs\":[],\"obfuscation\":\"eCqgHs\",\"output_index\":0,\"sequence_number\":10}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" a\",\"item_id\":\"msg_04d28ba03840bb65016a705b7358a081a38e69464057840036\",\"logprobs\":[],\"obfuscation\":\"7RAJkk4N6FYOim\",\"output_index\":0,\"sequence_number\":11}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" step\",\"item_id\":\"msg_04d28ba03840bb65016a705b7358a081a38e69464057840036\",\"logprobs\":[],\"obfuscation\":\"y7scL3Od23b\",\"output_index\":0,\"sequence_number\":12}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"-by\",\"item_id\":\"msg_04d28ba03840bb65016a705b7358a081a38e69464057840036\",\"logprobs\":[],\"obfuscation\":\"W7lrv8VHEOIfi\",\"output_index\":0,\"sequence_number\":13}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"-step\",\"item_id\":\"msg_04d28ba03840bb65016a705b7358a081a38e69464057840036\",\"logprobs\":[],\"obfuscation\":\"4LIgLNAJAB0\",\"output_index\":0,\"sequence_number\":14}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" execution\",\"item_id\":\"msg_04d28ba03840bb65016a705b7358a081a38e69464057840036\",\"logprobs\":[],\"obfuscation\":\"F077yL\",\"output_index\":0,\"sequence_number\":15}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" of\",\"item_id\":\"msg_04d28ba03840bb65016a705b7358a081a38e69464057840036\",\"logprobs\":[],\"obfuscation\":\"fMOsa5wMUpP8t\",\"output_index\":0,\"sequence_number\":16}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" an\",\"item_id\":\"msg_04d28ba03840bb65016a705b7358a081a38e69464057840036\",\"logprobs\":[],\"obfuscation\":\"SXWFdQci5VvIi\",\"output_index\":0,\"sequence_number\":17}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" instrument\",\"item_id\":\"msg_04d28ba03840bb65016a705b7358a081a38e69464057840036\",\"logprobs\":[],\"obfuscation\":\"IM33q\",\"output_index\":0,\"sequence_number\":18}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"ed\",\"item_id\":\"msg_04d28ba03840bb65016a705b7358a081a38e69464057840036\",\"logprobs\":[],\"obfuscation\":\"nO8YVpkRDW8oOQ\",\"output_index\":0,\"sequence_number\":19}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" research\",\"item_id\":\"msg_04d28ba03840bb65016a705b7358a081a38e69464057840036\",\"logprobs\":[],\"obfuscation\":\"xjbeOFx\",\"output_index\":0,\"sequence_number\":20}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" flow\",\"item_id\":\"msg_04d28ba03840bb65016a705b7358a081a38e69464057840036\",\"logprobs\":[],\"obfuscation\":\"PjjjDiy3SBJ\",\"output_index\":0,\"sequence_number\":21}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" involving\",\"item_id\":\"msg_04d28ba03840bb65016a705b7358a081a38e69464057840036\",\"logprobs\":[],\"obfuscation\":\"I58VnM\",\"output_index\":0,\"sequence_number\":22}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" tool\",\"item_id\":\"msg_04d28ba03840bb65016a705b7358a081a38e69464057840036\",\"logprobs\":[],\"obfuscation\":\"i0bH5Hefbkz\",\"output_index\":0,\"sequence_number\":23}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" calls\",\"item_id\":\"msg_04d28ba03840bb65016a705b7358a081a38e69464057840036\",\"logprobs\":[],\"obfuscation\":\"BEMMaOeJJU\",\"output_index\":0,\"sequence_number\":24}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" to\",\"item_id\":\"msg_04d28ba03840bb65016a705b7358a081a38e69464057840036\",\"logprobs\":[],\"obfuscation\":\"gPeC3aszPMi0S\",\"output_index\":0,\"sequence_number\":25}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" gather\",\"item_id\":\"msg_04d28ba03840bb65016a705b7358a081a38e69464057840036\",\"logprobs\":[],\"obfuscation\":\"zlkkxcR5g\",\"output_index\":0,\"sequence_number\":26}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" information\",\"item_id\":\"msg_04d28ba03840bb65016a705b7358a081a38e69464057840036\",\"logprobs\":[],\"obfuscation\":\"7hhs\",\"output_index\":0,\"sequence_number\":27}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" on\",\"item_id\":\"msg_04d28ba03840bb65016a705b7358a081a38e69464057840036\",\"logprobs\":[],\"obfuscation\":\"lLr5hZIXvG8Gb\",\"output_index\":0,\"sequence_number\":28}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" \\\"\",\"item_id\":\"msg_04d28ba03840bb65016a705b7358a081a38e69464057840036\",\"logprobs\":[],\"obfuscation\":\"5qdl5MlY2bxs7F\",\"output_index\":0,\"sequence_number\":29}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"fl\",\"item_id\":\"msg_04d28ba03840bb65016a705b7358a081a38e69464057840036\",\"logprobs\":[],\"obfuscation\":\"yt6z8GDsVV0Qew\",\"output_index\":0,\"sequence_number\":30}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"ue\",\"item_id\":\"msg_04d28ba03840bb65016a705b7358a081a38e69464057840036\",\"logprobs\":[],\"obfuscation\":\"3QmzDfnfod8ZRQ\",\"output_index\":0,\"sequence_number\":31}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" instrumentation\",\"item_id\":\"msg_04d28ba03840bb65016a705b7358a081a38e69464057840036\",\"logprobs\":[],\"obfuscation\":\"\",\"output_index\":0,\"sequence_number\":32}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"\\\"\",\"item_id\":\"msg_04d28ba03840bb65016a705b7358a081a38e69464057840036\",\"logprobs\":[],\"obfuscation\":\"yiSjj02o3FbcC4U\",\"output_index\":0,\"sequence_number\":33}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" and\",\"item_id\":\"msg_04d28ba03840bb65016a705b7358a081a38e69464057840036\",\"logprobs\":[],\"obfuscation\":\"796Vjv2p8Npd\",\"output_index\":0,\"sequence_number\":34}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" \\\"\",\"item_id\":\"msg_04d28ba03840bb65016a705b7358a081a38e69464057840036\",\"logprobs\":[],\"obfuscation\":\"WkdXPw3c1tG4kc\",\"output_index\":0,\"sequence_number\":35}", - "event: response.output_text.done\ndata: {\"type\":\"response.output_text.done\",\"content_index\":0,\"item_id\":\"msg_04d28ba03840bb65016a705b7358a081a38e69464057840036\",\"logprobs\":[],\"output_index\":0,\"sequence_number\":36,\"text\":\"## Original Request\\nThe user requested a step-by-step execution of an instrumented research flow involving tool calls to gather information on \\\"flue instrumentation\\\" and \\\"\"}", - "event: response.content_part.done\ndata: {\"type\":\"response.content_part.done\",\"content_index\":0,\"item_id\":\"msg_04d28ba03840bb65016a705b7358a081a38e69464057840036\",\"output_index\":0,\"part\":{\"type\":\"output_text\",\"annotations\":[],\"logprobs\":[],\"text\":\"## Original Request\\nThe user requested a step-by-step execution of an instrumented research flow involving tool calls to gather information on \\\"flue instrumentation\\\" and \\\"\"},\"sequence_number\":37}", - "event: response.output_item.done\ndata: {\"type\":\"response.output_item.done\",\"item\":{\"id\":\"msg_04d28ba03840bb65016a705b7358a081a38e69464057840036\",\"type\":\"message\",\"status\":\"incomplete\",\"content\":[{\"type\":\"output_text\",\"annotations\":[],\"logprobs\":[],\"text\":\"## Original Request\\nThe user requested a step-by-step execution of an instrumented research flow involving tool calls to gather information on \\\"flue instrumentation\\\" and \\\"\"}],\"role\":\"assistant\"},\"output_index\":0,\"sequence_number\":38}", - "event: response.incomplete\ndata: {\"type\":\"response.incomplete\",\"response\":{\"id\":\"resp_04d28ba03840bb65016a705b72d36081a3a387e30f2739466c\",\"object\":\"response\",\"created_at\":1785748338,\"status\":\"incomplete\",\"background\":false,\"completed_at\":null,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":{\"reason\":\"max_output_tokens\"},\"instructions\":null,\"max_output_tokens\":32,\"max_tool_calls\":null,\"model\":\"gpt-4o-mini-2024-07-18\",\"moderation\":null,\"output\":[{\"id\":\"msg_04d28ba03840bb65016a705b7358a081a38e69464057840036\",\"type\":\"message\",\"status\":\"incomplete\",\"content\":[{\"type\":\"output_text\",\"annotations\":[],\"logprobs\":[],\"text\":\"## Original Request\\nThe user requested a step-by-step execution of an instrumented research flow involving tool calls to gather information on \\\"flue instrumentation\\\" and \\\"\"}],\"role\":\"assistant\"}],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":null,\"prompt_cache_retention\":\"in_memory\",\"reasoning\":{\"context\":null,\"effort\":null,\"summary\":null},\"safety_identifier\":null,\"service_tier\":\"default\",\"store\":false,\"temperature\":1,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tool_usage\":{\"image_gen\":{\"input_tokens\":0,\"input_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"output_tokens\":0,\"output_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"total_tokens\":0},\"web_search\":{\"num_requests\":0}},\"tools\":[],\"top_logprobs\":0,\"top_p\":1,\"truncation\":\"disabled\",\"usage\":{\"input_tokens\":446,\"input_tokens_details\":{\"cache_write_tokens\":0,\"cached_tokens\":0},\"output_tokens\":32,\"output_tokens_details\":{\"reasoning_tokens\":0},\"total_tokens\":478},\"user\":null,\"metadata\":{}},\"sequence_number\":39}" - ], - "kind": "sse" - }, - "headers": { - "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", - "alt-svc": "h3=\":443\"; ma=86400", - "cf-cache-status": "DYNAMIC", - "cf-ray": "a254332d68c17bb6-IAD", - "connection": "keep-alive", - "content-type": "text/event-stream; charset=utf-8", - "date": "Mon, 03 Aug 2026 09:12:18 GMT", - "openai-organization": "braintrust-data", - "openai-processing-ms": "159", - "openai-project": "proj_vsCSXafhhByzWOThMrJcZiw9", - "openai-version": "2020-10-01", - "server": "cloudflare", - "set-cookie": "[REDACTED]", - "strict-transport-security": "max-age=31536000; includeSubDomains; preload", - "transfer-encoding": "chunked", - "x-content-type-options": "nosniff", - "x-request-id": "req_7ea02cb4cc7c43e08845985aea9af023" - }, - "status": 200, - "statusText": "OK" - } - } - ], - "meta": { - "createdAt": "2026-08-03T09:07:11.649Z" - } -} diff --git a/e2e/scenarios/flue-instrumentation/__snapshots__/flue-v0-8-0-auto-hook.span-tree.json b/e2e/scenarios/flue-instrumentation/__snapshots__/flue-v0-8-0-auto-hook.span-tree.json deleted file mode 100644 index dd92abe29..000000000 --- a/e2e/scenarios/flue-instrumentation/__snapshots__/flue-v0-8-0-auto-hook.span-tree.json +++ /dev/null @@ -1,1448 +0,0 @@ -{ - "span_tree": [ - { - "name": "workflow:instrumentation", - "type": "task", - "children": [ - { - "name": "flue.skill", - "type": "task", - "children": [ - { - "name": "flue.turn", - "type": "llm", - "children": [], - "input": [ - { - "content": [ - { - "text": "Run the skill named \"e2e-flue-skill\".\n\nArguments:\n{\n \"marker\": \"SKILL_DONE\"\n}", - "type": "text" - } - ], - "role": "user" - } - ], - "output": { - "content": [ - { - "text": "I'll run the e2e-flue-skill. Let me first read the skill file to understand its instructions.", - "type": "text" - }, - { - "arguments": { - "path": ".agents/e2e-flue-skill.md" - }, - "id": "", - "name": "read", - "type": "toolCall" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "anthropic-messages", - "flue.model": "claude-sonnet-4-5-20250929", - "flue.provider": "anthropic", - "flue.session": "skill", - "flue.stop_reason": "toolUse", - "flue.turn_purpose": "agent", - "model": "claude-sonnet-4-5-20250929", - "provider": "anthropic" - }, - "metrics": { - "completion_tokens": 89, - "duration_ms": 0, - "estimated_cost": 0.0041862, - "prompt_cache_creation_tokens": 670, - "prompt_cached_tokens": 1099, - "prompt_tokens": 3, - "tokens": 1861 - } - }, - { - "name": "tool:read", - "type": "tool", - "children": [], - "input": { - "path": ".agents/e2e-flue-skill.md" - }, - "output": { - "content": [ - { - "text": "ENOENT: no such file or directory, open '/e2e/scenarios/flue-instrumentation/.agents/e2e-flue-skill.md'", - "type": "text" - } - ], - "details": {} - }, - "metadata": { - "flue.session": "skill", - "flue.tool_name": "read", - "provider": "flue" - }, - "metrics": { - "duration_ms": 0 - }, - "error": "{\"content\":[{\"type\":\"text\",\"text\":\"ENOENT: no such file or directory, open '/e2e/scenarios/flue-instrumentation/.agents/e2e-flue-skill.md'\"}],\"details\":{}}" - }, - { - "name": "flue.turn", - "type": "llm", - "children": [], - "input": [ - { - "content": [ - { - "text": "Run the skill named \"e2e-flue-skill\".\n\nArguments:\n{\n \"marker\": \"SKILL_DONE\"\n}", - "type": "text" - } - ], - "role": "user" - }, - { - "content": [ - { - "text": "I'll run the e2e-flue-skill. Let me first read the skill file to understand its instructions.", - "type": "text" - }, - { - "arguments": { - "path": ".agents/e2e-flue-skill.md" - }, - "id": "", - "name": "read", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "ENOENT: no such file or directory, open '/e2e/scenarios/flue-instrumentation/.agents/e2e-flue-skill.md'", - "type": "text" - } - ], - "isError": true, - "role": "toolResult", - "toolCallId": "", - "toolName": "read" - } - ], - "output": { - "content": [ - { - "arguments": { - "path": ".agents" - }, - "id": "", - "name": "read", - "type": "toolCall" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "anthropic-messages", - "flue.model": "claude-sonnet-4-5-20250929", - "flue.provider": "anthropic", - "flue.session": "skill", - "flue.stop_reason": "toolUse", - "flue.turn_purpose": "agent", - "model": "claude-sonnet-4-5-20250929", - "provider": "anthropic" - }, - "metrics": { - "completion_tokens": 52, - "duration_ms": 0, - "estimated_cost": 0.00201795, - "prompt_cache_creation_tokens": 183, - "prompt_cached_tokens": 1769, - "prompt_tokens": 7, - "tokens": 2011 - } - }, - { - "name": "tool:read", - "type": "tool", - "children": [], - "input": { - "path": ".agents" - }, - "output": { - "content": [ - { - "text": "skills", - "type": "text" - } - ], - "details": { - "entries": 1, - "isDirectory": true, - "path": ".agents" - } - }, - "metadata": { - "flue.session": "skill", - "flue.tool_name": "read", - "provider": "flue" - }, - "metrics": { - "duration_ms": 0 - } - }, - { - "name": "flue.turn", - "type": "llm", - "children": [], - "input": [ - { - "content": [ - { - "text": "Run the skill named \"e2e-flue-skill\".\n\nArguments:\n{\n \"marker\": \"SKILL_DONE\"\n}", - "type": "text" - } - ], - "role": "user" - }, - { - "content": [ - { - "text": "I'll run the e2e-flue-skill. Let me first read the skill file to understand its instructions.", - "type": "text" - }, - { - "arguments": { - "path": ".agents/e2e-flue-skill.md" - }, - "id": "", - "name": "read", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "ENOENT: no such file or directory, open '/e2e/scenarios/flue-instrumentation/.agents/e2e-flue-skill.md'", - "type": "text" - } - ], - "isError": true, - "role": "toolResult", - "toolCallId": "", - "toolName": "read" - }, - { - "content": [ - { - "arguments": { - "path": ".agents" - }, - "id": "", - "name": "read", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "skills", - "type": "text" - } - ], - "isError": false, - "role": "toolResult", - "toolCallId": "", - "toolName": "read" - } - ], - "output": { - "content": [ - { - "arguments": { - "path": ".agents/skills" - }, - "id": "", - "name": "read", - "type": "toolCall" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "anthropic-messages", - "flue.model": "claude-sonnet-4-5-20250929", - "flue.provider": "anthropic", - "flue.session": "skill", - "flue.stop_reason": "toolUse", - "flue.turn_purpose": "agent", - "model": "claude-sonnet-4-5-20250929", - "provider": "anthropic" - }, - "metrics": { - "completion_tokens": 54, - "duration_ms": 0, - "estimated_cost": 0.0016611, - "prompt_cache_creation_tokens": 66, - "prompt_cached_tokens": 1952, - "prompt_tokens": 6, - "tokens": 2078 - } - }, - { - "name": "tool:read", - "type": "tool", - "children": [], - "input": { - "path": ".agents/skills" - }, - "output": { - "content": [ - { - "text": "e2e-flue-skill", - "type": "text" - } - ], - "details": { - "entries": 1, - "isDirectory": true, - "path": ".agents/skills" - } - }, - "metadata": { - "flue.session": "skill", - "flue.tool_name": "read", - "provider": "flue" - }, - "metrics": { - "duration_ms": 0 - } - }, - { - "name": "flue.turn", - "type": "llm", - "children": [], - "input": [ - { - "content": [ - { - "text": "Run the skill named \"e2e-flue-skill\".\n\nArguments:\n{\n \"marker\": \"SKILL_DONE\"\n}", - "type": "text" - } - ], - "role": "user" - }, - { - "content": [ - { - "text": "I'll run the e2e-flue-skill. Let me first read the skill file to understand its instructions.", - "type": "text" - }, - { - "arguments": { - "path": ".agents/e2e-flue-skill.md" - }, - "id": "", - "name": "read", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "ENOENT: no such file or directory, open '/e2e/scenarios/flue-instrumentation/.agents/e2e-flue-skill.md'", - "type": "text" - } - ], - "isError": true, - "role": "toolResult", - "toolCallId": "", - "toolName": "read" - }, - { - "content": [ - { - "arguments": { - "path": ".agents" - }, - "id": "", - "name": "read", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "skills", - "type": "text" - } - ], - "isError": false, - "role": "toolResult", - "toolCallId": "", - "toolName": "read" - }, - { - "content": [ - { - "arguments": { - "path": ".agents/skills" - }, - "id": "", - "name": "read", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "e2e-flue-skill", - "type": "text" - } - ], - "isError": false, - "role": "toolResult", - "toolCallId": "", - "toolName": "read" - } - ], - "output": { - "content": [ - { - "arguments": { - "path": ".agents/skills/e2e-flue-skill" - }, - "id": "", - "name": "read", - "type": "toolCall" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "anthropic-messages", - "flue.model": "claude-sonnet-4-5-20250929", - "flue.provider": "anthropic", - "flue.session": "skill", - "flue.stop_reason": "toolUse", - "flue.turn_purpose": "agent", - "model": "claude-sonnet-4-5-20250929", - "provider": "anthropic" - }, - "metrics": { - "completion_tokens": 63, - "duration_ms": 0, - "estimated_cost": 0.0018459, - "prompt_cache_creation_tokens": 74, - "prompt_cached_tokens": 2018, - "prompt_tokens": 6, - "tokens": 2161 - } - }, - { - "name": "tool:read", - "type": "tool", - "children": [], - "input": { - "path": ".agents/skills/e2e-flue-skill" - }, - "output": { - "content": [ - { - "text": "SKILL.md", - "type": "text" - } - ], - "details": { - "entries": 1, - "isDirectory": true, - "path": ".agents/skills/e2e-flue-skill" - } - }, - "metadata": { - "flue.session": "skill", - "flue.tool_name": "read", - "provider": "flue" - }, - "metrics": { - "duration_ms": 0 - } - }, - { - "name": "flue.turn", - "type": "llm", - "children": [], - "input": [ - { - "content": [ - { - "text": "Run the skill named \"e2e-flue-skill\".\n\nArguments:\n{\n \"marker\": \"SKILL_DONE\"\n}", - "type": "text" - } - ], - "role": "user" - }, - { - "content": [ - { - "text": "I'll run the e2e-flue-skill. Let me first read the skill file to understand its instructions.", - "type": "text" - }, - { - "arguments": { - "path": ".agents/e2e-flue-skill.md" - }, - "id": "", - "name": "read", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "ENOENT: no such file or directory, open '/e2e/scenarios/flue-instrumentation/.agents/e2e-flue-skill.md'", - "type": "text" - } - ], - "isError": true, - "role": "toolResult", - "toolCallId": "", - "toolName": "read" - }, - { - "content": [ - { - "arguments": { - "path": ".agents" - }, - "id": "", - "name": "read", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "skills", - "type": "text" - } - ], - "isError": false, - "role": "toolResult", - "toolCallId": "", - "toolName": "read" - }, - { - "content": [ - { - "arguments": { - "path": ".agents/skills" - }, - "id": "", - "name": "read", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "e2e-flue-skill", - "type": "text" - } - ], - "isError": false, - "role": "toolResult", - "toolCallId": "", - "toolName": "read" - }, - { - "content": [ - { - "arguments": { - "path": ".agents/skills/e2e-flue-skill" - }, - "id": "", - "name": "read", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "SKILL.md", - "type": "text" - } - ], - "isError": false, - "role": "toolResult", - "toolCallId": "", - "toolName": "read" - } - ], - "output": { - "content": [ - { - "arguments": { - "path": ".agents/skills/e2e-flue-skill/SKILL.md" - }, - "id": "", - "name": "read", - "type": "toolCall" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "anthropic-messages", - "flue.model": "claude-sonnet-4-5-20250929", - "flue.provider": "anthropic", - "flue.session": "skill", - "flue.stop_reason": "toolUse", - "flue.turn_purpose": "agent", - "model": "claude-sonnet-4-5-20250929", - "provider": "anthropic" - }, - "metrics": { - "completion_tokens": 68, - "duration_ms": 0, - "estimated_cost": 0.0019618500000000002, - "prompt_cache_creation_tokens": 79, - "prompt_cached_tokens": 2092, - "prompt_tokens": 6, - "tokens": 2245 - } - }, - { - "name": "tool:read", - "type": "tool", - "children": [], - "input": { - "path": ".agents/skills/e2e-flue-skill/SKILL.md" - }, - "output": { - "content": [ - { - "text": "---\nname: e2e-flue-skill\ndescription: Deterministic skill used by the Braintrust Flue instrumentation e2e test.\n---\n\nReturn the marker from the provided args exactly once. Output no other text.\n", - "type": "text" - } - ], - "details": { - "lines": 7, - "path": ".agents/skills/e2e-flue-skill/SKILL.md" - } - }, - "metadata": { - "flue.session": "skill", - "flue.tool_name": "read", - "provider": "flue" - }, - "metrics": { - "duration_ms": 0 - } - }, - { - "name": "flue.turn", - "type": "llm", - "children": [], - "input": [ - { - "content": [ - { - "text": "Run the skill named \"e2e-flue-skill\".\n\nArguments:\n{\n \"marker\": \"SKILL_DONE\"\n}", - "type": "text" - } - ], - "role": "user" - }, - { - "content": [ - { - "text": "I'll run the e2e-flue-skill. Let me first read the skill file to understand its instructions.", - "type": "text" - }, - { - "arguments": { - "path": ".agents/e2e-flue-skill.md" - }, - "id": "", - "name": "read", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "ENOENT: no such file or directory, open '/e2e/scenarios/flue-instrumentation/.agents/e2e-flue-skill.md'", - "type": "text" - } - ], - "isError": true, - "role": "toolResult", - "toolCallId": "", - "toolName": "read" - }, - { - "content": [ - { - "arguments": { - "path": ".agents" - }, - "id": "", - "name": "read", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "skills", - "type": "text" - } - ], - "isError": false, - "role": "toolResult", - "toolCallId": "", - "toolName": "read" - }, - { - "content": [ - { - "arguments": { - "path": ".agents/skills" - }, - "id": "", - "name": "read", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "e2e-flue-skill", - "type": "text" - } - ], - "isError": false, - "role": "toolResult", - "toolCallId": "", - "toolName": "read" - }, - { - "content": [ - { - "arguments": { - "path": ".agents/skills/e2e-flue-skill" - }, - "id": "", - "name": "read", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "SKILL.md", - "type": "text" - } - ], - "isError": false, - "role": "toolResult", - "toolCallId": "", - "toolName": "read" - }, - { - "content": [ - { - "arguments": { - "path": ".agents/skills/e2e-flue-skill/SKILL.md" - }, - "id": "", - "name": "read", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "---\nname: e2e-flue-skill\ndescription: Deterministic skill used by the Braintrust Flue instrumentation e2e test.\n---\n\nReturn the marker from the provided args exactly once. Output no other text.\n", - "type": "text" - } - ], - "isError": false, - "role": "toolResult", - "toolCallId": "", - "toolName": "read" - } - ], - "output": { - "content": [ - { - "text": "SKILL_DONE", - "type": "text" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "anthropic-messages", - "flue.model": "claude-sonnet-4-5-20250929", - "flue.provider": "anthropic", - "flue.session": "skill", - "flue.stop_reason": "stop", - "flue.turn_purpose": "agent", - "model": "claude-sonnet-4-5-20250929", - "provider": "anthropic" - }, - "metrics": { - "completion_tokens": 8, - "duration_ms": 0, - "estimated_cost": 0.00129555, - "prompt_cache_creation_tokens": 135, - "prompt_cached_tokens": 2171, - "prompt_tokens": 6, - "tokens": 2320 - } - } - ], - "input": [ - { - "content": [ - { - "text": "Run the skill named \"e2e-flue-skill\".\n\nArguments:\n{\n \"marker\": \"SKILL_DONE\"\n}", - "type": "text" - } - ], - "role": "user" - } - ], - "output": "SKILL_DONE", - "metadata": { - "flue.operation": "skill", - "flue.session": "skill", - "flue.workflow_name": "instrumentation", - "provider": "flue", - "scenario": "flue-instrumentation" - }, - "metrics": { - "duration_ms": 0 - } - }, - { - "name": "flue.task", - "type": "task", - "children": [ - { - "name": "flue.turn", - "type": "llm", - "children": [], - "input": [ - { - "content": [ - { - "text": "Reply with exactly TASK_DONE and no other text.", - "type": "text" - } - ], - "role": "user" - } - ], - "output": { - "content": [ - { - "text": "TASK_DONE", - "textSignature": "{\"v\":1,\"id\":\"msg_09b32b2821e9a4d8016a184600019081929d34bb7d4c6a246f\"}", - "type": "text" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "openai-responses", - "flue.input_mode": "full", - "flue.model": "gpt-4o-mini", - "flue.provider": "openai", - "flue.session": "task:task:", - "flue.stop_reason": "stop", - "flue.turn_purpose": "agent", - "model": "gpt-4o-mini", - "provider": "openai" - }, - "metrics": { - "completion_tokens": 4, - "duration_ms": 0, - "estimated_cost": 0.00012299999999999998, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 804, - "tokens": 808 - } - } - ], - "input": "Reply with exactly TASK_DONE and no other text.", - "output": "TASK_DONE", - "metadata": { - "flue.session": "task", - "provider": "flue" - }, - "metrics": { - "duration_ms": 0 - } - }, - { - "name": "flue.compact", - "type": "task", - "children": [ - { - "name": "compaction:manual", - "type": "task", - "children": [ - { - "name": "flue.turn", - "type": "llm", - "children": [], - "input": [ - { - "content": [ - { - "text": "\n[User]: Complete this instrumented research flow. Call exactly one tool per turn and wait for each tool result before choosing the next tool. Step 1: call lookup with query \"flue instrumentation\". Step 2: use the lookup result id as lookupId and call web_search with query \"Braintrust Flue reasoning stream instrumentation\". Step 3: use the first web_search result url and call summarize_source. After summarize_source returns, reply with exactly PROMPT_DONE and no other text.\n\n[Assistant thinking]: The user wants me to complete a research flow with exactly three steps:\n1. Call lookup with query \"flue instrumentation\"\n2. Call web_search with the lookupId from step 1 and query \"Braintrust Flue reasoning stream instrumentation\"\n3. Call summarize_source with the first URL from step 2\n4. Reply with exactly \"PROMPT_DONE\"\n\nThe user emphasizes I should call exactly one tool per turn and wait for each result. Let me start with step 1.\n\n[Assistant]: I'll complete this instrumented research flow step by step. Starting with Step 1:\n\n[Assistant tool calls]: lookup(query=\"flue instrumentation\")\n\n[Tool result]: {\"id\":\"flue-session-2026\",\"query\":\"flue instrumentation\",\"topic\":\"session instrumentation\"}\n\n[Assistant thinking]: Good, I got the lookup result with id \"flue-session-2026\". Now I need to proceed to step 2: call web_search with this lookupId and the query \"Braintrust Flue reasoning stream instrumentation\".\n\n[Assistant]: Now proceeding to Step 2:\n\n[Assistant tool calls]: web_search(lookupId=\"flue-session-2026\", query=\"Braintrust Flue reasoning stream instrumentation\")\n\n[Tool result]: {\"lookupId\":\"flue-session-2026\",\"query\":\"Braintrust Flue reasoning stream instrumentation\",\"results\":[{\"title\":\"Flue reasoning stream instrumentation\",\"url\":\"https://example.test/flue/reasoning-streams\"}]}\n\n[Assistant thinking]: Perfect! I got the web_search results with one result. The first (and only) result has the URL \"https://example.test/flue/reasoning-streams\". Now I need to proceed to step 3: call summarize_source with this URL.\n\n[Assistant]: Now proceeding to Step 3:\n\n[Assistant tool calls]: summarize_source(url=\"https://example.test/flue/reasoning-streams\")\n\n[Tool result]: {\"summary\":\"Flue emits reasoning, tool execution, and LLM turn events separately.\",\"url\":\"https://example.test/flue/reasoning-streams\"}\n\n\nThis is the PREFIX of a turn that was too large to keep. The SUFFIX (recent work) is retained.\n\nSummarize the prefix to provide context for the retained suffix:\n\n## Original Request\n[What did the user ask for in this turn?]\n\n## Early Progress\n- [Key decisions and work done in the prefix]\n\n## Context for Suffix\n- [Information needed to understand the retained recent work]\n\nBe concise. Focus on what's needed to understand the kept suffix.", - "type": "text" - } - ], - "role": "user" - } - ], - "output": { - "content": [ - { - "text": "## Original Request\nThe user requested to complete an instrumented research flow consisting of three specific steps, involving calling various tools in a sequential manner.\n\n## Early Progress", - "textSignature": "{\"v\":1,\"id\":\"msg_01ecf68325f9e202016a184601309081a0878ef4054e950d32\"}", - "type": "text" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "openai-responses", - "flue.model": "gpt-4o-mini", - "flue.provider": "openai", - "flue.session": "main", - "flue.stop_reason": "stop", - "flue.turn_purpose": "compaction_prefix", - "model": "gpt-4o-mini", - "provider": "openai" - }, - "metrics": { - "completion_tokens": 0, - "duration_ms": 0, - "estimated_cost": 0, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 0, - "tokens": 0 - } - } - ], - "input": { - "estimatedTokens": 2950, - "reason": "manual" - }, - "output": { - "messagesAfter": 2, - "messagesBefore": 8 - }, - "metadata": { - "flue.compaction_reason": "manual", - "flue.session": "main", - "provider": "flue" - }, - "metrics": { - "duration_ms": 0, - "messages_after": 2, - "messages_before": 8 - } - } - ], - "input": { - "estimatedTokens": 2950, - "reason": "manual" - }, - "output": { - "completed": true - }, - "metadata": { - "flue.operation": "compact", - "flue.session": "main", - "flue.workflow_name": "instrumentation", - "provider": "flue", - "scenario": "flue-instrumentation" - }, - "metrics": { - "duration_ms": 0 - } - } - ], - "input": { - "metadata": { - "scenario": "flue-instrumentation", - "testRunId": "" - }, - "scenario": "flue-instrumentation" - }, - "output": { - "scenario": "flue-instrumentation", - "status": "done" - }, - "metadata": { - "flue.workflow_name": "instrumentation", - "provider": "flue", - "scenario": "flue-instrumentation" - }, - "metrics": { - "duration_ms": 0 - } - }, - { - "name": "flue.prompt", - "type": "task", - "children": [ - { - "name": "flue.turn", - "type": "llm", - "children": [], - "input": [ - { - "content": [ - { - "text": "Complete this instrumented research flow. Call exactly one tool per turn and wait for each tool result before choosing the next tool. Step 1: call lookup with query \"flue instrumentation\". Step 2: use the lookup result id as lookupId and call web_search with query \"Braintrust Flue reasoning stream instrumentation\". Step 3: use the first web_search result url and call summarize_source. After summarize_source returns, reply with exactly PROMPT_DONE and no other text.", - "type": "text" - } - ], - "role": "user" - } - ], - "output": { - "content": [ - { - "thinking": "The user wants me to complete a research flow with exactly three steps:\n1. Call lookup with query \"flue instrumentation\"\n2. Call web_search with the lookupId from step 1 and query \"Braintrust Flue reasoning stream instrumentation\"\n3. Call summarize_source with the first URL from step 2\n4. Reply with exactly \"PROMPT_DONE\"\n\nThe user emphasizes I should call exactly one tool per turn and wait for each result. Let me start with step 1.", - "thinkingSignature": "EoQFCm4IDhgCKkD+AoKMhpXNPUJ5eANwM28uUCJPCE+AXh0ACw7aXniX+YN04zdJl3hlHMbtAuKhhczLiEeOJtu4KtVpea5u5NFdMhpjbGF1ZGUtc29ubmV0LTQtNS0yMDI1MDkyOTgAQgh0aGlua2luZxIMIOTWCNEtsiI9vF4SGgwPXC861uZx8lpOziEiMLsCukGIluD0U44degzYnHFc+BwXNIEB+yatSHlZE+a5smr9crZcft3OLmsIdXuaFSrDAyQxWd9oqfJdnvza9eWIaOCeNkVCjrTc/wotBwh5Xp8m9k/WKt7P4zzUJBuLQ1f6ZnsdWkL8OiL/2E7Up55z2+UuVWsg1Y90/x27YRFdQjv3RnztYp4bZ5oIHAlTJoCzWiBYEh/Z8c2O+wy6MHo1cp0T57OroB/4HAqRZxrjTOXpKYLmYKXAY5tU8DTd3A0/FfEhg3oPSbOhX+112szFCpm3BRNJYooUUhwSu1hO4H/6r7d/Yp9XIM4tBfzuDx83CC/YetdOFgnYkdliw0xaUGaj4o37rWIRctWHAMdlUc4NDA0Ljc1uirHDXkN8EJwWSoIjPSBTA/p/wgLrvRV5Yy9Z0dMoWOpgYF6t1H/tNAyY8ml1CA7IWyd3jnfTZ1/G3Y0xFDwJDFlb4/QKc67w36SSCa+X9mMoTxD/DYN249lsII+XX3lQ7F1zJ07THOtCyRirCaRhmE/kGwl5tiB1QHrWa9bNHxQxf9i92435JGKN053Fe14AR70MtBSzIU2/zbgvhB1vF5lZt2i46hNxfo4lXR495gEwdwC6+8HJnlEJAJkdeHfZKj+RiwXoiYKPKsSbHi83X8Hh32IifuglNxdlxVQYAQ==", - "type": "thinking" - }, - { - "text": "I'll complete this instrumented research flow step by step. Starting with Step 1:", - "type": "text" - }, - { - "arguments": { - "query": "flue instrumentation" - }, - "id": "", - "name": "lookup", - "type": "toolCall" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "anthropic-messages", - "flue.input_mode": "full", - "flue.model": "claude-sonnet-4-5-20250929", - "flue.provider": "anthropic", - "flue.session": "main", - "flue.stop_reason": "toolUse", - "flue.turn_purpose": "agent", - "model": "claude-sonnet-4-5-20250929", - "provider": "anthropic" - }, - "metrics": { - "completion_tokens": 196, - "duration_ms": 0, - "estimated_cost": 0.0113025, - "prompt_cache_creation_tokens": 2222, - "prompt_cached_tokens": 0, - "prompt_tokens": 10, - "tokens": 2428 - } - }, - { - "name": "tool:lookup", - "type": "tool", - "children": [], - "input": { - "query": "flue instrumentation" - }, - "output": { - "content": [ - { - "text": "{\"id\":\"flue-session-2026\",\"query\":\"flue instrumentation\",\"topic\":\"session instrumentation\"}", - "type": "text" - } - ], - "details": { - "customTool": "lookup" - } - }, - "metadata": { - "flue.session": "main", - "flue.tool_name": "lookup", - "provider": "flue" - }, - "metrics": { - "duration_ms": 0 - } - }, - { - "name": "flue.turn", - "type": "llm", - "children": [], - "input": [ - { - "content": [ - { - "thinking": "The user wants me to complete a research flow with exactly three steps:\n1. Call lookup with query \"flue instrumentation\"\n2. Call web_search with the lookupId from step 1 and query \"Braintrust Flue reasoning stream instrumentation\"\n3. Call summarize_source with the first URL from step 2\n4. Reply with exactly \"PROMPT_DONE\"\n\nThe user emphasizes I should call exactly one tool per turn and wait for each result. Let me start with step 1.", - "thinkingSignature": "EoQFCm4IDhgCKkD+AoKMhpXNPUJ5eANwM28uUCJPCE+AXh0ACw7aXniX+YN04zdJl3hlHMbtAuKhhczLiEeOJtu4KtVpea5u5NFdMhpjbGF1ZGUtc29ubmV0LTQtNS0yMDI1MDkyOTgAQgh0aGlua2luZxIMIOTWCNEtsiI9vF4SGgwPXC861uZx8lpOziEiMLsCukGIluD0U44degzYnHFc+BwXNIEB+yatSHlZE+a5smr9crZcft3OLmsIdXuaFSrDAyQxWd9oqfJdnvza9eWIaOCeNkVCjrTc/wotBwh5Xp8m9k/WKt7P4zzUJBuLQ1f6ZnsdWkL8OiL/2E7Up55z2+UuVWsg1Y90/x27YRFdQjv3RnztYp4bZ5oIHAlTJoCzWiBYEh/Z8c2O+wy6MHo1cp0T57OroB/4HAqRZxrjTOXpKYLmYKXAY5tU8DTd3A0/FfEhg3oPSbOhX+112szFCpm3BRNJYooUUhwSu1hO4H/6r7d/Yp9XIM4tBfzuDx83CC/YetdOFgnYkdliw0xaUGaj4o37rWIRctWHAMdlUc4NDA0Ljc1uirHDXkN8EJwWSoIjPSBTA/p/wgLrvRV5Yy9Z0dMoWOpgYF6t1H/tNAyY8ml1CA7IWyd3jnfTZ1/G3Y0xFDwJDFlb4/QKc67w36SSCa+X9mMoTxD/DYN249lsII+XX3lQ7F1zJ07THOtCyRirCaRhmE/kGwl5tiB1QHrWa9bNHxQxf9i92435JGKN053Fe14AR70MtBSzIU2/zbgvhB1vF5lZt2i46hNxfo4lXR495gEwdwC6+8HJnlEJAJkdeHfZKj+RiwXoiYKPKsSbHi83X8Hh32IifuglNxdlxVQYAQ==", - "type": "thinking" - }, - { - "text": "I'll complete this instrumented research flow step by step. Starting with Step 1:", - "type": "text" - }, - { - "arguments": { - "query": "flue instrumentation" - }, - "id": "", - "name": "lookup", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "{\"id\":\"flue-session-2026\",\"query\":\"flue instrumentation\",\"topic\":\"session instrumentation\"}", - "type": "text" - } - ], - "isError": false, - "role": "toolResult", - "toolCallId": "", - "toolName": "lookup" - } - ], - "output": { - "content": [ - { - "thinking": "Good, I got the lookup result with id \"flue-session-2026\". Now I need to proceed to step 2: call web_search with this lookupId and the query \"Braintrust Flue reasoning stream instrumentation\".", - "thinkingSignature": "EpEDCm4IDhgCKkAMahve1kB1j4njp1ZSzO8eCK4bb5Ex7MB6tjdoOXfuUDRfkBqvIGdwVly8GiA8GGYC7Y/RVfREXF6A6OXHTwFyMhpjbGF1ZGUtc29ubmV0LTQtNS0yMDI1MDkyOTgAQgh0aGlua2luZxIMLw+fsa4yo7a7ZSoIGgyaTdvH020n2dQj9bUiMB4OokY29w+uZb4rqDnKDbLXUqUj6U0f1QCGQF2ZBYDDfDJyM5Jv1xY3rQiUUjkhBirQAS4XivjvMnWI5zHBmpI/u/pq4aGdrdb3vUsJ/ne6R1d80JA/INGF8vLkQcZ5xxEK0jU4yRE+ldc2uzavmtoxIKAFD6bqKks1Qv6u+TIZtuW7tUB/7jFxDw93KmJNruMIa8c1J6pPUyBy3xE7QwgmU0kDfIfdnljxLoEBiMwtnxjCPj9lT7HHz4FkrhbnQTV4MvAQohCLjaSbOTd6LdLxwtJPm9mc26ID4eYN1gCCzyAxGXhqLHlzc23PafvQjsZliWnLhaiNMnG0YEo78XkuJQsYAQ==", - "type": "thinking" - }, - { - "text": "Now proceeding to Step 2:", - "type": "text" - }, - { - "arguments": { - "lookupId": "flue-session-2026", - "query": "Braintrust Flue reasoning stream instrumentation" - }, - "id": "", - "name": "web_search", - "type": "toolCall" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "anthropic-messages", - "flue.input_message_offset": 1, - "flue.input_mode": "delta", - "flue.model": "claude-sonnet-4-5-20250929", - "flue.provider": "anthropic", - "flue.session": "main", - "flue.stop_reason": "toolUse", - "flue.turn_purpose": "agent", - "model": "claude-sonnet-4-5-20250929", - "provider": "anthropic" - }, - "metrics": { - "completion_tokens": 158, - "duration_ms": 0, - "estimated_cost": 0.0039606, - "prompt_cache_creation_tokens": 236, - "prompt_cached_tokens": 2222, - "prompt_tokens": 13, - "tokens": 2629 - } - }, - { - "name": "tool:web_search", - "type": "tool", - "children": [], - "input": { - "lookupId": "flue-session-2026", - "query": "Braintrust Flue reasoning stream instrumentation" - }, - "output": { - "content": [ - { - "text": "{\"lookupId\":\"flue-session-2026\",\"query\":\"Braintrust Flue reasoning stream instrumentation\",\"results\":[{\"title\":\"Flue reasoning stream instrumentation\",\"url\":\"https://example.test/flue/reasoning-streams\"}]}", - "type": "text" - } - ], - "details": { - "customTool": "web_search" - } - }, - "metadata": { - "flue.session": "main", - "flue.tool_name": "web_search", - "provider": "flue" - }, - "metrics": { - "duration_ms": 0 - } - }, - { - "name": "flue.turn", - "type": "llm", - "children": [], - "input": [ - { - "content": [ - { - "thinking": "Good, I got the lookup result with id \"flue-session-2026\". Now I need to proceed to step 2: call web_search with this lookupId and the query \"Braintrust Flue reasoning stream instrumentation\".", - "thinkingSignature": "EpEDCm4IDhgCKkAMahve1kB1j4njp1ZSzO8eCK4bb5Ex7MB6tjdoOXfuUDRfkBqvIGdwVly8GiA8GGYC7Y/RVfREXF6A6OXHTwFyMhpjbGF1ZGUtc29ubmV0LTQtNS0yMDI1MDkyOTgAQgh0aGlua2luZxIMLw+fsa4yo7a7ZSoIGgyaTdvH020n2dQj9bUiMB4OokY29w+uZb4rqDnKDbLXUqUj6U0f1QCGQF2ZBYDDfDJyM5Jv1xY3rQiUUjkhBirQAS4XivjvMnWI5zHBmpI/u/pq4aGdrdb3vUsJ/ne6R1d80JA/INGF8vLkQcZ5xxEK0jU4yRE+ldc2uzavmtoxIKAFD6bqKks1Qv6u+TIZtuW7tUB/7jFxDw93KmJNruMIa8c1J6pPUyBy3xE7QwgmU0kDfIfdnljxLoEBiMwtnxjCPj9lT7HHz4FkrhbnQTV4MvAQohCLjaSbOTd6LdLxwtJPm9mc26ID4eYN1gCCzyAxGXhqLHlzc23PafvQjsZliWnLhaiNMnG0YEo78XkuJQsYAQ==", - "type": "thinking" - }, - { - "text": "Now proceeding to Step 2:", - "type": "text" - }, - { - "arguments": { - "lookupId": "flue-session-2026", - "query": "Braintrust Flue reasoning stream instrumentation" - }, - "id": "", - "name": "web_search", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "{\"lookupId\":\"flue-session-2026\",\"query\":\"Braintrust Flue reasoning stream instrumentation\",\"results\":[{\"title\":\"Flue reasoning stream instrumentation\",\"url\":\"https://example.test/flue/reasoning-streams\"}]}", - "type": "text" - } - ], - "isError": false, - "role": "toolResult", - "toolCallId": "", - "toolName": "web_search" - } - ], - "output": { - "content": [ - { - "thinking": "Perfect! I got the web_search results with one result. The first (and only) result has the URL \"https://example.test/flue/reasoning-streams\". Now I need to proceed to step 3: call summarize_source with this URL.", - "thinkingSignature": "EqQDCm4IDhgCKkAXv7EDAhWMcrBz11bUd5qxlzyy03CAGQfRgLHDE0uKzLATh1o2sedd8GxLwwQqV0CmLRfNxsdK/ZDp3Lks9iysMhpjbGF1ZGUtc29ubmV0LTQtNS0yMDI1MDkyOTgAQgh0aGlua2luZxIMXDMniOy5Dy0KI8x7GgzgHs7BYJMs6/N0ur8iMPe0SwnHhP39OjEu7P8ASwK9ffwa3VP67QOlPzOstRRdEZNd0mq+BVjpmBXB+QcAzCrjAeAh/UO/d7SfFmXjqwA3//thoL/TTlGa6W+fdOCFRikn65JWT4ZdVFetfqD+9/QCP/nXEwMsLnTBAxC4EGryrgs2YT+OWwmEJ7rHKFeDs5djhwSb2J2ZJO1kRvD9s2vCCDkImygYBSIVgx1Y0CZJHhzn2Vf39v0Fbth2wDZa+OqxivV9NmDZEjmgcM+tTRx1CCnaww8LZEJQ68v/dUJy7hhD4MrNcQ62evJYPfT3+4EKKWJ/7HI7KXz91//+YAkph63XstbJMFJ3AldAFiq3ecauXt5r42N5gmCmwZX+sI48ENLGGAE=", - "type": "thinking" - }, - { - "text": "Now proceeding to Step 3:", - "type": "text" - }, - { - "arguments": { - "url": "https://example.test/flue/reasoning-streams" - }, - "id": "", - "name": "summarize_source", - "type": "toolCall" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "anthropic-messages", - "flue.input_message_offset": 3, - "flue.input_mode": "delta", - "flue.model": "claude-sonnet-4-5-20250929", - "flue.provider": "anthropic", - "flue.session": "main", - "flue.stop_reason": "toolUse", - "flue.turn_purpose": "agent", - "model": "claude-sonnet-4-5-20250929", - "provider": "anthropic" - }, - "metrics": { - "completion_tokens": 138, - "duration_ms": 0, - "estimated_cost": 0.0037239000000000005, - "prompt_cache_creation_tokens": 234, - "prompt_cached_tokens": 2458, - "prompt_tokens": 13, - "tokens": 2843 - } - }, - { - "name": "tool:summarize_source", - "type": "tool", - "children": [], - "input": { - "url": "https://example.test/flue/reasoning-streams" - }, - "output": { - "content": [ - { - "text": "{\"summary\":\"Flue emits reasoning, tool execution, and LLM turn events separately.\",\"url\":\"https://example.test/flue/reasoning-streams\"}", - "type": "text" - } - ], - "details": { - "customTool": "summarize_source" - } - }, - "metadata": { - "flue.session": "main", - "flue.tool_name": "summarize_source", - "provider": "flue" - }, - "metrics": { - "duration_ms": 0 - } - }, - { - "name": "flue.turn", - "type": "llm", - "children": [], - "input": [ - { - "content": [ - { - "thinking": "Perfect! I got the web_search results with one result. The first (and only) result has the URL \"https://example.test/flue/reasoning-streams\". Now I need to proceed to step 3: call summarize_source with this URL.", - "thinkingSignature": "EqQDCm4IDhgCKkAXv7EDAhWMcrBz11bUd5qxlzyy03CAGQfRgLHDE0uKzLATh1o2sedd8GxLwwQqV0CmLRfNxsdK/ZDp3Lks9iysMhpjbGF1ZGUtc29ubmV0LTQtNS0yMDI1MDkyOTgAQgh0aGlua2luZxIMXDMniOy5Dy0KI8x7GgzgHs7BYJMs6/N0ur8iMPe0SwnHhP39OjEu7P8ASwK9ffwa3VP67QOlPzOstRRdEZNd0mq+BVjpmBXB+QcAzCrjAeAh/UO/d7SfFmXjqwA3//thoL/TTlGa6W+fdOCFRikn65JWT4ZdVFetfqD+9/QCP/nXEwMsLnTBAxC4EGryrgs2YT+OWwmEJ7rHKFeDs5djhwSb2J2ZJO1kRvD9s2vCCDkImygYBSIVgx1Y0CZJHhzn2Vf39v0Fbth2wDZa+OqxivV9NmDZEjmgcM+tTRx1CCnaww8LZEJQ68v/dUJy7hhD4MrNcQ62evJYPfT3+4EKKWJ/7HI7KXz91//+YAkph63XstbJMFJ3AldAFiq3ecauXt5r42N5gmCmwZX+sI48ENLGGAE=", - "type": "thinking" - }, - { - "text": "Now proceeding to Step 3:", - "type": "text" - }, - { - "arguments": { - "url": "https://example.test/flue/reasoning-streams" - }, - "id": "", - "name": "summarize_source", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "{\"summary\":\"Flue emits reasoning, tool execution, and LLM turn events separately.\",\"url\":\"https://example.test/flue/reasoning-streams\"}", - "type": "text" - } - ], - "isError": false, - "role": "toolResult", - "toolCallId": "", - "toolName": "summarize_source" - } - ], - "output": { - "content": [ - { - "thinking": "Perfect! The summarize_source call has completed successfully. According to the instructions, I should now reply with exactly \"PROMPT_DONE\" and no other text.", - "thinkingSignature": "Eu8CCm4IDhgCKkAn2QOIIlYfvcs6ynsmhCslHDtBsrTA4CfmimK7zyz6+k6dZldV2JBtEBET41It0gFcGEKC0n0mAhaYVuLutpDDMhpjbGF1ZGUtc29ubmV0LTQtNS0yMDI1MDkyOTgAQgh0aGlua2luZxIM1xfLptxSF8rKzou8GgwRLuVmfiBYOT4CZIsiMAYz8TpvchMAWodP+T9uZ+dJ47MVdNv+p86nZXmT5d5EWlCroiAOFHHzJH2o0UjJQSquAX6wu+jIC8/DXwurGIIf2ToldPuJWF5HGj8kHMnwh777XS1NUFl1irM24rvP7kTAv3Dp8YEwb8t3g1EsicY7j+C5DzgwxVsPYCVOCgdMqQjtsqpL7AX5KQ9+hR1gK1DQjshv4Gze3hw/+C+9iptd3RlDIuwO0cm8NY+sVqeWVh3TtQujQlVt1rW/HfWjclrtT0LZSGwc6RypmfMMQi6lncIk/EPgy0XbS1plKW0tLBgB", - "type": "thinking" - }, - { - "text": "PROMPT_DONE", - "type": "text" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "anthropic-messages", - "flue.input_message_offset": 5, - "flue.input_mode": "delta", - "flue.model": "claude-sonnet-4-5-20250929", - "flue.provider": "anthropic", - "flue.session": "main", - "flue.stop_reason": "stop", - "flue.turn_purpose": "agent", - "model": "claude-sonnet-4-5-20250929", - "provider": "anthropic" - }, - "metrics": { - "completion_tokens": 51, - "duration_ms": 0, - "estimated_cost": 0.0023391, - "prompt_cache_creation_tokens": 194, - "prompt_cached_tokens": 2692, - "prompt_tokens": 13, - "tokens": 2950 - } - } - ], - "input": [ - { - "content": [ - { - "text": "Complete this instrumented research flow. Call exactly one tool per turn and wait for each tool result before choosing the next tool. Step 1: call lookup with query \"flue instrumentation\". Step 2: use the lookup result id as lookupId and call web_search with query \"Braintrust Flue reasoning stream instrumentation\". Step 3: use the first web_search result url and call summarize_source. After summarize_source returns, reply with exactly PROMPT_DONE and no other text.", - "type": "text" - } - ], - "role": "user" - } - ], - "output": "PROMPT_DONE", - "metadata": { - "flue.operation": "prompt", - "flue.session": "main", - "flue.workflow_name": "instrumentation", - "provider": "flue", - "scenario": "flue-instrumentation" - }, - "metrics": { - "duration_ms": 0 - } - } - ] -} diff --git a/e2e/scenarios/flue-instrumentation/__snapshots__/flue-v0-8-0-auto-hook.span-tree.txt b/e2e/scenarios/flue-instrumentation/__snapshots__/flue-v0-8-0-auto-hook.span-tree.txt deleted file mode 100644 index e90a4892a..000000000 --- a/e2e/scenarios/flue-instrumentation/__snapshots__/flue-v0-8-0-auto-hook.span-tree.txt +++ /dev/null @@ -1,1335 +0,0 @@ -span_tree: -├── workflow:instrumentation [task] -│ input: { -│ "metadata": { -│ "scenario": "flue-instrumentation", -│ "testRunId": "" -│ }, -│ "scenario": "flue-instrumentation" -│ } -│ output: { -│ "scenario": "flue-instrumentation", -│ "status": "done" -│ } -│ metadata: { -│ "flue.workflow_name": "instrumentation", -│ "provider": "flue", -│ "scenario": "flue-instrumentation" -│ } -│ metrics: { -│ "duration_ms": 0 -│ } -│ ├── flue.skill [task] -│ │ input: [ -│ │ { -│ │ "content": [ -│ │ { -│ │ "text": "Run the skill named \"e2e-flue-skill\".\n\nArguments:\n{\n \"marker\": \"SKILL_DONE\"\n}", -│ │ "type": "text" -│ │ } -│ │ ], -│ │ "role": "user" -│ │ } -│ │ ] -│ │ output: "SKILL_DONE" -│ │ metadata: { -│ │ "flue.operation": "skill", -│ │ "flue.session": "skill", -│ │ "flue.workflow_name": "instrumentation", -│ │ "provider": "flue", -│ │ "scenario": "flue-instrumentation" -│ │ } -│ │ metrics: { -│ │ "duration_ms": 0 -│ │ } -│ │ ├── flue.turn [llm] -│ │ │ input: [ -│ │ │ { -│ │ │ "content": [ -│ │ │ { -│ │ │ "text": "Run the skill named \"e2e-flue-skill\".\n\nArguments:\n{\n \"marker\": \"SKILL_DONE\"\n}", -│ │ │ "type": "text" -│ │ │ } -│ │ │ ], -│ │ │ "role": "user" -│ │ │ } -│ │ │ ] -│ │ │ output: { -│ │ │ "content": [ -│ │ │ { -│ │ │ "text": "I'll run the e2e-flue-skill. Let me first read the skill file to understand its instructions.", -│ │ │ "type": "text" -│ │ │ }, -│ │ │ { -│ │ │ "arguments": { -│ │ │ "path": ".agents/e2e-flue-skill.md" -│ │ │ }, -│ │ │ "id": "", -│ │ │ "name": "read", -│ │ │ "type": "toolCall" -│ │ │ } -│ │ │ ], -│ │ │ "role": "assistant" -│ │ │ } -│ │ │ metadata: { -│ │ │ "flue.api": "anthropic-messages", -│ │ │ "flue.model": "claude-sonnet-4-5-20250929", -│ │ │ "flue.provider": "anthropic", -│ │ │ "flue.session": "skill", -│ │ │ "flue.stop_reason": "toolUse", -│ │ │ "flue.turn_purpose": "agent", -│ │ │ "model": "claude-sonnet-4-5-20250929", -│ │ │ "provider": "anthropic" -│ │ │ } -│ │ │ metrics: { -│ │ │ "completion_tokens": 89, -│ │ │ "duration_ms": 0, -│ │ │ "estimated_cost": 0.0041862, -│ │ │ "prompt_cache_creation_tokens": 670, -│ │ │ "prompt_cached_tokens": 1099, -│ │ │ "prompt_tokens": 3, -│ │ │ "tokens": 1861 -│ │ │ } -│ │ ├── tool:read [tool] -│ │ │ input: { -│ │ │ "path": ".agents/e2e-flue-skill.md" -│ │ │ } -│ │ │ output: { -│ │ │ "content": [ -│ │ │ { -│ │ │ "text": "ENOENT: no such file or directory, open '/e2e/scenarios/flue-instrumentation/.agents/e2e-flue-skill.md'", -│ │ │ "type": "text" -│ │ │ } -│ │ │ ], -│ │ │ "details": {} -│ │ │ } -│ │ │ metadata: { -│ │ │ "flue.session": "skill", -│ │ │ "flue.tool_name": "read", -│ │ │ "provider": "flue" -│ │ │ } -│ │ │ metrics: { -│ │ │ "duration_ms": 0 -│ │ │ } -│ │ │ error: "{\"content\":[{\"type\":\"text\",\"text\":\"ENOENT: no such file or directory, open '/e2e/scenarios/flue-instrumentation/.agents/e2e-flue-skill.md'\"}],\"details\":{}}" -│ │ ├── flue.turn [llm] -│ │ │ input: [ -│ │ │ { -│ │ │ "content": [ -│ │ │ { -│ │ │ "text": "Run the skill named \"e2e-flue-skill\".\n\nArguments:\n{\n \"marker\": \"SKILL_DONE\"\n}", -│ │ │ "type": "text" -│ │ │ } -│ │ │ ], -│ │ │ "role": "user" -│ │ │ }, -│ │ │ { -│ │ │ "content": [ -│ │ │ { -│ │ │ "text": "I'll run the e2e-flue-skill. Let me first read the skill file to understand its instructions.", -│ │ │ "type": "text" -│ │ │ }, -│ │ │ { -│ │ │ "arguments": { -│ │ │ "path": ".agents/e2e-flue-skill.md" -│ │ │ }, -│ │ │ "id": "", -│ │ │ "name": "read", -│ │ │ "type": "toolCall" -│ │ │ } -│ │ │ ], -│ │ │ "role": "assistant" -│ │ │ }, -│ │ │ { -│ │ │ "content": [ -│ │ │ { -│ │ │ "text": "ENOENT: no such file or directory, open '/e2e/scenarios/flue-instrumentation/.agents/e2e-flue-skill.md'", -│ │ │ "type": "text" -│ │ │ } -│ │ │ ], -│ │ │ "isError": true, -│ │ │ "role": "toolResult", -│ │ │ "toolCallId": "", -│ │ │ "toolName": "read" -│ │ │ } -│ │ │ ] -│ │ │ output: { -│ │ │ "content": [ -│ │ │ { -│ │ │ "arguments": { -│ │ │ "path": ".agents" -│ │ │ }, -│ │ │ "id": "", -│ │ │ "name": "read", -│ │ │ "type": "toolCall" -│ │ │ } -│ │ │ ], -│ │ │ "role": "assistant" -│ │ │ } -│ │ │ metadata: { -│ │ │ "flue.api": "anthropic-messages", -│ │ │ "flue.model": "claude-sonnet-4-5-20250929", -│ │ │ "flue.provider": "anthropic", -│ │ │ "flue.session": "skill", -│ │ │ "flue.stop_reason": "toolUse", -│ │ │ "flue.turn_purpose": "agent", -│ │ │ "model": "claude-sonnet-4-5-20250929", -│ │ │ "provider": "anthropic" -│ │ │ } -│ │ │ metrics: { -│ │ │ "completion_tokens": 52, -│ │ │ "duration_ms": 0, -│ │ │ "estimated_cost": 0.00201795, -│ │ │ "prompt_cache_creation_tokens": 183, -│ │ │ "prompt_cached_tokens": 1769, -│ │ │ "prompt_tokens": 7, -│ │ │ "tokens": 2011 -│ │ │ } -│ │ ├── tool:read [tool] -│ │ │ input: { -│ │ │ "path": ".agents" -│ │ │ } -│ │ │ output: { -│ │ │ "content": [ -│ │ │ { -│ │ │ "text": "skills", -│ │ │ "type": "text" -│ │ │ } -│ │ │ ], -│ │ │ "details": { -│ │ │ "entries": 1, -│ │ │ "isDirectory": true, -│ │ │ "path": ".agents" -│ │ │ } -│ │ │ } -│ │ │ metadata: { -│ │ │ "flue.session": "skill", -│ │ │ "flue.tool_name": "read", -│ │ │ "provider": "flue" -│ │ │ } -│ │ │ metrics: { -│ │ │ "duration_ms": 0 -│ │ │ } -│ │ ├── flue.turn [llm] -│ │ │ input: [ -│ │ │ { -│ │ │ "content": [ -│ │ │ { -│ │ │ "text": "Run the skill named \"e2e-flue-skill\".\n\nArguments:\n{\n \"marker\": \"SKILL_DONE\"\n}", -│ │ │ "type": "text" -│ │ │ } -│ │ │ ], -│ │ │ "role": "user" -│ │ │ }, -│ │ │ { -│ │ │ "content": [ -│ │ │ { -│ │ │ "text": "I'll run the e2e-flue-skill. Let me first read the skill file to understand its instructions.", -│ │ │ "type": "text" -│ │ │ }, -│ │ │ { -│ │ │ "arguments": { -│ │ │ "path": ".agents/e2e-flue-skill.md" -│ │ │ }, -│ │ │ "id": "", -│ │ │ "name": "read", -│ │ │ "type": "toolCall" -│ │ │ } -│ │ │ ], -│ │ │ "role": "assistant" -│ │ │ }, -│ │ │ { -│ │ │ "content": [ -│ │ │ { -│ │ │ "text": "ENOENT: no such file or directory, open '/e2e/scenarios/flue-instrumentation/.agents/e2e-flue-skill.md'", -│ │ │ "type": "text" -│ │ │ } -│ │ │ ], -│ │ │ "isError": true, -│ │ │ "role": "toolResult", -│ │ │ "toolCallId": "", -│ │ │ "toolName": "read" -│ │ │ }, -│ │ │ { -│ │ │ "content": [ -│ │ │ { -│ │ │ "arguments": { -│ │ │ "path": ".agents" -│ │ │ }, -│ │ │ "id": "", -│ │ │ "name": "read", -│ │ │ "type": "toolCall" -│ │ │ } -│ │ │ ], -│ │ │ "role": "assistant" -│ │ │ }, -│ │ │ { -│ │ │ "content": [ -│ │ │ { -│ │ │ "text": "skills", -│ │ │ "type": "text" -│ │ │ } -│ │ │ ], -│ │ │ "isError": false, -│ │ │ "role": "toolResult", -│ │ │ "toolCallId": "", -│ │ │ "toolName": "read" -│ │ │ } -│ │ │ ] -│ │ │ output: { -│ │ │ "content": [ -│ │ │ { -│ │ │ "arguments": { -│ │ │ "path": ".agents/skills" -│ │ │ }, -│ │ │ "id": "", -│ │ │ "name": "read", -│ │ │ "type": "toolCall" -│ │ │ } -│ │ │ ], -│ │ │ "role": "assistant" -│ │ │ } -│ │ │ metadata: { -│ │ │ "flue.api": "anthropic-messages", -│ │ │ "flue.model": "claude-sonnet-4-5-20250929", -│ │ │ "flue.provider": "anthropic", -│ │ │ "flue.session": "skill", -│ │ │ "flue.stop_reason": "toolUse", -│ │ │ "flue.turn_purpose": "agent", -│ │ │ "model": "claude-sonnet-4-5-20250929", -│ │ │ "provider": "anthropic" -│ │ │ } -│ │ │ metrics: { -│ │ │ "completion_tokens": 54, -│ │ │ "duration_ms": 0, -│ │ │ "estimated_cost": 0.0016611, -│ │ │ "prompt_cache_creation_tokens": 66, -│ │ │ "prompt_cached_tokens": 1952, -│ │ │ "prompt_tokens": 6, -│ │ │ "tokens": 2078 -│ │ │ } -│ │ ├── tool:read [tool] -│ │ │ input: { -│ │ │ "path": ".agents/skills" -│ │ │ } -│ │ │ output: { -│ │ │ "content": [ -│ │ │ { -│ │ │ "text": "e2e-flue-skill", -│ │ │ "type": "text" -│ │ │ } -│ │ │ ], -│ │ │ "details": { -│ │ │ "entries": 1, -│ │ │ "isDirectory": true, -│ │ │ "path": ".agents/skills" -│ │ │ } -│ │ │ } -│ │ │ metadata: { -│ │ │ "flue.session": "skill", -│ │ │ "flue.tool_name": "read", -│ │ │ "provider": "flue" -│ │ │ } -│ │ │ metrics: { -│ │ │ "duration_ms": 0 -│ │ │ } -│ │ ├── flue.turn [llm] -│ │ │ input: [ -│ │ │ { -│ │ │ "content": [ -│ │ │ { -│ │ │ "text": "Run the skill named \"e2e-flue-skill\".\n\nArguments:\n{\n \"marker\": \"SKILL_DONE\"\n}", -│ │ │ "type": "text" -│ │ │ } -│ │ │ ], -│ │ │ "role": "user" -│ │ │ }, -│ │ │ { -│ │ │ "content": [ -│ │ │ { -│ │ │ "text": "I'll run the e2e-flue-skill. Let me first read the skill file to understand its instructions.", -│ │ │ "type": "text" -│ │ │ }, -│ │ │ { -│ │ │ "arguments": { -│ │ │ "path": ".agents/e2e-flue-skill.md" -│ │ │ }, -│ │ │ "id": "", -│ │ │ "name": "read", -│ │ │ "type": "toolCall" -│ │ │ } -│ │ │ ], -│ │ │ "role": "assistant" -│ │ │ }, -│ │ │ { -│ │ │ "content": [ -│ │ │ { -│ │ │ "text": "ENOENT: no such file or directory, open '/e2e/scenarios/flue-instrumentation/.agents/e2e-flue-skill.md'", -│ │ │ "type": "text" -│ │ │ } -│ │ │ ], -│ │ │ "isError": true, -│ │ │ "role": "toolResult", -│ │ │ "toolCallId": "", -│ │ │ "toolName": "read" -│ │ │ }, -│ │ │ { -│ │ │ "content": [ -│ │ │ { -│ │ │ "arguments": { -│ │ │ "path": ".agents" -│ │ │ }, -│ │ │ "id": "", -│ │ │ "name": "read", -│ │ │ "type": "toolCall" -│ │ │ } -│ │ │ ], -│ │ │ "role": "assistant" -│ │ │ }, -│ │ │ { -│ │ │ "content": [ -│ │ │ { -│ │ │ "text": "skills", -│ │ │ "type": "text" -│ │ │ } -│ │ │ ], -│ │ │ "isError": false, -│ │ │ "role": "toolResult", -│ │ │ "toolCallId": "", -│ │ │ "toolName": "read" -│ │ │ }, -│ │ │ { -│ │ │ "content": [ -│ │ │ { -│ │ │ "arguments": { -│ │ │ "path": ".agents/skills" -│ │ │ }, -│ │ │ "id": "", -│ │ │ "name": "read", -│ │ │ "type": "toolCall" -│ │ │ } -│ │ │ ], -│ │ │ "role": "assistant" -│ │ │ }, -│ │ │ { -│ │ │ "content": [ -│ │ │ { -│ │ │ "text": "e2e-flue-skill", -│ │ │ "type": "text" -│ │ │ } -│ │ │ ], -│ │ │ "isError": false, -│ │ │ "role": "toolResult", -│ │ │ "toolCallId": "", -│ │ │ "toolName": "read" -│ │ │ } -│ │ │ ] -│ │ │ output: { -│ │ │ "content": [ -│ │ │ { -│ │ │ "arguments": { -│ │ │ "path": ".agents/skills/e2e-flue-skill" -│ │ │ }, -│ │ │ "id": "", -│ │ │ "name": "read", -│ │ │ "type": "toolCall" -│ │ │ } -│ │ │ ], -│ │ │ "role": "assistant" -│ │ │ } -│ │ │ metadata: { -│ │ │ "flue.api": "anthropic-messages", -│ │ │ "flue.model": "claude-sonnet-4-5-20250929", -│ │ │ "flue.provider": "anthropic", -│ │ │ "flue.session": "skill", -│ │ │ "flue.stop_reason": "toolUse", -│ │ │ "flue.turn_purpose": "agent", -│ │ │ "model": "claude-sonnet-4-5-20250929", -│ │ │ "provider": "anthropic" -│ │ │ } -│ │ │ metrics: { -│ │ │ "completion_tokens": 63, -│ │ │ "duration_ms": 0, -│ │ │ "estimated_cost": 0.0018459, -│ │ │ "prompt_cache_creation_tokens": 74, -│ │ │ "prompt_cached_tokens": 2018, -│ │ │ "prompt_tokens": 6, -│ │ │ "tokens": 2161 -│ │ │ } -│ │ ├── tool:read [tool] -│ │ │ input: { -│ │ │ "path": ".agents/skills/e2e-flue-skill" -│ │ │ } -│ │ │ output: { -│ │ │ "content": [ -│ │ │ { -│ │ │ "text": "SKILL.md", -│ │ │ "type": "text" -│ │ │ } -│ │ │ ], -│ │ │ "details": { -│ │ │ "entries": 1, -│ │ │ "isDirectory": true, -│ │ │ "path": ".agents/skills/e2e-flue-skill" -│ │ │ } -│ │ │ } -│ │ │ metadata: { -│ │ │ "flue.session": "skill", -│ │ │ "flue.tool_name": "read", -│ │ │ "provider": "flue" -│ │ │ } -│ │ │ metrics: { -│ │ │ "duration_ms": 0 -│ │ │ } -│ │ ├── flue.turn [llm] -│ │ │ input: [ -│ │ │ { -│ │ │ "content": [ -│ │ │ { -│ │ │ "text": "Run the skill named \"e2e-flue-skill\".\n\nArguments:\n{\n \"marker\": \"SKILL_DONE\"\n}", -│ │ │ "type": "text" -│ │ │ } -│ │ │ ], -│ │ │ "role": "user" -│ │ │ }, -│ │ │ { -│ │ │ "content": [ -│ │ │ { -│ │ │ "text": "I'll run the e2e-flue-skill. Let me first read the skill file to understand its instructions.", -│ │ │ "type": "text" -│ │ │ }, -│ │ │ { -│ │ │ "arguments": { -│ │ │ "path": ".agents/e2e-flue-skill.md" -│ │ │ }, -│ │ │ "id": "", -│ │ │ "name": "read", -│ │ │ "type": "toolCall" -│ │ │ } -│ │ │ ], -│ │ │ "role": "assistant" -│ │ │ }, -│ │ │ { -│ │ │ "content": [ -│ │ │ { -│ │ │ "text": "ENOENT: no such file or directory, open '/e2e/scenarios/flue-instrumentation/.agents/e2e-flue-skill.md'", -│ │ │ "type": "text" -│ │ │ } -│ │ │ ], -│ │ │ "isError": true, -│ │ │ "role": "toolResult", -│ │ │ "toolCallId": "", -│ │ │ "toolName": "read" -│ │ │ }, -│ │ │ { -│ │ │ "content": [ -│ │ │ { -│ │ │ "arguments": { -│ │ │ "path": ".agents" -│ │ │ }, -│ │ │ "id": "", -│ │ │ "name": "read", -│ │ │ "type": "toolCall" -│ │ │ } -│ │ │ ], -│ │ │ "role": "assistant" -│ │ │ }, -│ │ │ { -│ │ │ "content": [ -│ │ │ { -│ │ │ "text": "skills", -│ │ │ "type": "text" -│ │ │ } -│ │ │ ], -│ │ │ "isError": false, -│ │ │ "role": "toolResult", -│ │ │ "toolCallId": "", -│ │ │ "toolName": "read" -│ │ │ }, -│ │ │ { -│ │ │ "content": [ -│ │ │ { -│ │ │ "arguments": { -│ │ │ "path": ".agents/skills" -│ │ │ }, -│ │ │ "id": "", -│ │ │ "name": "read", -│ │ │ "type": "toolCall" -│ │ │ } -│ │ │ ], -│ │ │ "role": "assistant" -│ │ │ }, -│ │ │ { -│ │ │ "content": [ -│ │ │ { -│ │ │ "text": "e2e-flue-skill", -│ │ │ "type": "text" -│ │ │ } -│ │ │ ], -│ │ │ "isError": false, -│ │ │ "role": "toolResult", -│ │ │ "toolCallId": "", -│ │ │ "toolName": "read" -│ │ │ }, -│ │ │ { -│ │ │ "content": [ -│ │ │ { -│ │ │ "arguments": { -│ │ │ "path": ".agents/skills/e2e-flue-skill" -│ │ │ }, -│ │ │ "id": "", -│ │ │ "name": "read", -│ │ │ "type": "toolCall" -│ │ │ } -│ │ │ ], -│ │ │ "role": "assistant" -│ │ │ }, -│ │ │ { -│ │ │ "content": [ -│ │ │ { -│ │ │ "text": "SKILL.md", -│ │ │ "type": "text" -│ │ │ } -│ │ │ ], -│ │ │ "isError": false, -│ │ │ "role": "toolResult", -│ │ │ "toolCallId": "", -│ │ │ "toolName": "read" -│ │ │ } -│ │ │ ] -│ │ │ output: { -│ │ │ "content": [ -│ │ │ { -│ │ │ "arguments": { -│ │ │ "path": ".agents/skills/e2e-flue-skill/SKILL.md" -│ │ │ }, -│ │ │ "id": "", -│ │ │ "name": "read", -│ │ │ "type": "toolCall" -│ │ │ } -│ │ │ ], -│ │ │ "role": "assistant" -│ │ │ } -│ │ │ metadata: { -│ │ │ "flue.api": "anthropic-messages", -│ │ │ "flue.model": "claude-sonnet-4-5-20250929", -│ │ │ "flue.provider": "anthropic", -│ │ │ "flue.session": "skill", -│ │ │ "flue.stop_reason": "toolUse", -│ │ │ "flue.turn_purpose": "agent", -│ │ │ "model": "claude-sonnet-4-5-20250929", -│ │ │ "provider": "anthropic" -│ │ │ } -│ │ │ metrics: { -│ │ │ "completion_tokens": 68, -│ │ │ "duration_ms": 0, -│ │ │ "estimated_cost": 0.0019618500000000002, -│ │ │ "prompt_cache_creation_tokens": 79, -│ │ │ "prompt_cached_tokens": 2092, -│ │ │ "prompt_tokens": 6, -│ │ │ "tokens": 2245 -│ │ │ } -│ │ ├── tool:read [tool] -│ │ │ input: { -│ │ │ "path": ".agents/skills/e2e-flue-skill/SKILL.md" -│ │ │ } -│ │ │ output: { -│ │ │ "content": [ -│ │ │ { -│ │ │ "text": "---\nname: e2e-flue-skill\ndescription: Deterministic skill used by the Braintrust Flue instrumentation e2e test.\n---\n\nReturn the marker from the provided args exactly once. Output no other text.\n", -│ │ │ "type": "text" -│ │ │ } -│ │ │ ], -│ │ │ "details": { -│ │ │ "lines": 7, -│ │ │ "path": ".agents/skills/e2e-flue-skill/SKILL.md" -│ │ │ } -│ │ │ } -│ │ │ metadata: { -│ │ │ "flue.session": "skill", -│ │ │ "flue.tool_name": "read", -│ │ │ "provider": "flue" -│ │ │ } -│ │ │ metrics: { -│ │ │ "duration_ms": 0 -│ │ │ } -│ │ └── flue.turn [llm] -│ │ input: [ -│ │ { -│ │ "content": [ -│ │ { -│ │ "text": "Run the skill named \"e2e-flue-skill\".\n\nArguments:\n{\n \"marker\": \"SKILL_DONE\"\n}", -│ │ "type": "text" -│ │ } -│ │ ], -│ │ "role": "user" -│ │ }, -│ │ { -│ │ "content": [ -│ │ { -│ │ "text": "I'll run the e2e-flue-skill. Let me first read the skill file to understand its instructions.", -│ │ "type": "text" -│ │ }, -│ │ { -│ │ "arguments": { -│ │ "path": ".agents/e2e-flue-skill.md" -│ │ }, -│ │ "id": "", -│ │ "name": "read", -│ │ "type": "toolCall" -│ │ } -│ │ ], -│ │ "role": "assistant" -│ │ }, -│ │ { -│ │ "content": [ -│ │ { -│ │ "text": "ENOENT: no such file or directory, open '/e2e/scenarios/flue-instrumentation/.agents/e2e-flue-skill.md'", -│ │ "type": "text" -│ │ } -│ │ ], -│ │ "isError": true, -│ │ "role": "toolResult", -│ │ "toolCallId": "", -│ │ "toolName": "read" -│ │ }, -│ │ { -│ │ "content": [ -│ │ { -│ │ "arguments": { -│ │ "path": ".agents" -│ │ }, -│ │ "id": "", -│ │ "name": "read", -│ │ "type": "toolCall" -│ │ } -│ │ ], -│ │ "role": "assistant" -│ │ }, -│ │ { -│ │ "content": [ -│ │ { -│ │ "text": "skills", -│ │ "type": "text" -│ │ } -│ │ ], -│ │ "isError": false, -│ │ "role": "toolResult", -│ │ "toolCallId": "", -│ │ "toolName": "read" -│ │ }, -│ │ { -│ │ "content": [ -│ │ { -│ │ "arguments": { -│ │ "path": ".agents/skills" -│ │ }, -│ │ "id": "", -│ │ "name": "read", -│ │ "type": "toolCall" -│ │ } -│ │ ], -│ │ "role": "assistant" -│ │ }, -│ │ { -│ │ "content": [ -│ │ { -│ │ "text": "e2e-flue-skill", -│ │ "type": "text" -│ │ } -│ │ ], -│ │ "isError": false, -│ │ "role": "toolResult", -│ │ "toolCallId": "", -│ │ "toolName": "read" -│ │ }, -│ │ { -│ │ "content": [ -│ │ { -│ │ "arguments": { -│ │ "path": ".agents/skills/e2e-flue-skill" -│ │ }, -│ │ "id": "", -│ │ "name": "read", -│ │ "type": "toolCall" -│ │ } -│ │ ], -│ │ "role": "assistant" -│ │ }, -│ │ { -│ │ "content": [ -│ │ { -│ │ "text": "SKILL.md", -│ │ "type": "text" -│ │ } -│ │ ], -│ │ "isError": false, -│ │ "role": "toolResult", -│ │ "toolCallId": "", -│ │ "toolName": "read" -│ │ }, -│ │ { -│ │ "content": [ -│ │ { -│ │ "arguments": { -│ │ "path": ".agents/skills/e2e-flue-skill/SKILL.md" -│ │ }, -│ │ "id": "", -│ │ "name": "read", -│ │ "type": "toolCall" -│ │ } -│ │ ], -│ │ "role": "assistant" -│ │ }, -│ │ { -│ │ "content": [ -│ │ { -│ │ "text": "---\nname: e2e-flue-skill\ndescription: Deterministic skill used by the Braintrust Flue instrumentation e2e test.\n---\n\nReturn the marker from the provided args exactly once. Output no other text.\n", -│ │ "type": "text" -│ │ } -│ │ ], -│ │ "isError": false, -│ │ "role": "toolResult", -│ │ "toolCallId": "", -│ │ "toolName": "read" -│ │ } -│ │ ] -│ │ output: { -│ │ "content": [ -│ │ { -│ │ "text": "SKILL_DONE", -│ │ "type": "text" -│ │ } -│ │ ], -│ │ "role": "assistant" -│ │ } -│ │ metadata: { -│ │ "flue.api": "anthropic-messages", -│ │ "flue.model": "claude-sonnet-4-5-20250929", -│ │ "flue.provider": "anthropic", -│ │ "flue.session": "skill", -│ │ "flue.stop_reason": "stop", -│ │ "flue.turn_purpose": "agent", -│ │ "model": "claude-sonnet-4-5-20250929", -│ │ "provider": "anthropic" -│ │ } -│ │ metrics: { -│ │ "completion_tokens": 8, -│ │ "duration_ms": 0, -│ │ "estimated_cost": 0.00129555, -│ │ "prompt_cache_creation_tokens": 135, -│ │ "prompt_cached_tokens": 2171, -│ │ "prompt_tokens": 6, -│ │ "tokens": 2320 -│ │ } -│ ├── flue.task [task] -│ │ input: "Reply with exactly TASK_DONE and no other text." -│ │ output: "TASK_DONE" -│ │ metadata: { -│ │ "flue.session": "task", -│ │ "provider": "flue" -│ │ } -│ │ metrics: { -│ │ "duration_ms": 0 -│ │ } -│ │ └── flue.turn [llm] -│ │ input: [ -│ │ { -│ │ "content": [ -│ │ { -│ │ "text": "Reply with exactly TASK_DONE and no other text.", -│ │ "type": "text" -│ │ } -│ │ ], -│ │ "role": "user" -│ │ } -│ │ ] -│ │ output: { -│ │ "content": [ -│ │ { -│ │ "text": "TASK_DONE", -│ │ "textSignature": "{\"v\":1,\"id\":\"msg_09b32b2821e9a4d8016a184600019081929d34bb7d4c6a246f\"}", -│ │ "type": "text" -│ │ } -│ │ ], -│ │ "role": "assistant" -│ │ } -│ │ metadata: { -│ │ "flue.api": "openai-responses", -│ │ "flue.input_mode": "full", -│ │ "flue.model": "gpt-4o-mini", -│ │ "flue.provider": "openai", -│ │ "flue.session": "task:task:", -│ │ "flue.stop_reason": "stop", -│ │ "flue.turn_purpose": "agent", -│ │ "model": "gpt-4o-mini", -│ │ "provider": "openai" -│ │ } -│ │ metrics: { -│ │ "completion_tokens": 4, -│ │ "duration_ms": 0, -│ │ "estimated_cost": 0.00012299999999999998, -│ │ "prompt_cache_creation_tokens": 0, -│ │ "prompt_cached_tokens": 0, -│ │ "prompt_tokens": 804, -│ │ "tokens": 808 -│ │ } -│ └── flue.compact [task] -│ input: { -│ "estimatedTokens": 2950, -│ "reason": "manual" -│ } -│ output: { -│ "completed": true -│ } -│ metadata: { -│ "flue.operation": "compact", -│ "flue.session": "main", -│ "flue.workflow_name": "instrumentation", -│ "provider": "flue", -│ "scenario": "flue-instrumentation" -│ } -│ metrics: { -│ "duration_ms": 0 -│ } -│ └── compaction:manual [task] -│ input: { -│ "estimatedTokens": 2950, -│ "reason": "manual" -│ } -│ output: { -│ "messagesAfter": 2, -│ "messagesBefore": 8 -│ } -│ metadata: { -│ "flue.compaction_reason": "manual", -│ "flue.session": "main", -│ "provider": "flue" -│ } -│ metrics: { -│ "duration_ms": 0, -│ "messages_after": 2, -│ "messages_before": 8 -│ } -│ └── flue.turn [llm] -│ input: [ -│ { -│ "content": [ -│ { -│ "text": "\n[User]: Complete this instrumented research flow. Call exactly one tool per turn and wait for each tool result before choosing the next tool. Step 1: call lookup with query \"flue instrumentation\". Step 2: use the lookup result id as lookupId and call web_search with query \"Braintrust Flue reasoning stream instrumentation\". Step 3: use the first web_search result url and call summarize_source. After summarize_source returns, reply with exactly PROMPT_DONE and no other text.\n\n[Assistant thinking]: The user wants me to complete a research flow with exactly three steps:\n1. Call lookup with query \"flue instrumentation\"\n2. Call web_search with the lookupId from step 1 and query \"Braintrust Flue reasoning stream instrumentation\"\n3. Call summarize_source with the first URL from step 2\n4. Reply with exactly \"PROMPT_DONE\"\n\nThe user emphasizes I should call exactly one tool per turn and wait for each result. Let me start with step 1.\n\n[Assistant]: I'll complete this instrumented research flow step by step. Starting with Step 1:\n\n[Assistant tool calls]: lookup(query=\"flue instrumentation\")\n\n[Tool result]: {\"id\":\"flue-session-2026\",\"query\":\"flue instrumentation\",\"topic\":\"session instrumentation\"}\n\n[Assistant thinking]: Good, I got the lookup result with id \"flue-session-2026\". Now I need to proceed to step 2: call web_search with this lookupId and the query \"Braintrust Flue reasoning stream instrumentation\".\n\n[Assistant]: Now proceeding to Step 2:\n\n[Assistant tool calls]: web_search(lookupId=\"flue-session-2026\", query=\"Braintrust Flue reasoning stream instrumentation\")\n\n[Tool result]: {\"lookupId\":\"flue-session-2026\",\"query\":\"Braintrust Flue reasoning stream instrumentation\",\"results\":[{\"title\":\"Flue reasoning stream instrumentation\",\"url\":\"https://example.test/flue/reasoning-streams\"}]}\n\n[Assistant thinking]: Perfect! I got the web_search results with one result. The first (and only) result has the URL \"https://example.test/flue/reasoning-streams\". Now I need to proceed to step 3: call summarize_source with this URL.\n\n[Assistant]: Now proceeding to Step 3:\n\n[Assistant tool calls]: summarize_source(url=\"https://example.test/flue/reasoning-streams\")\n\n[Tool result]: {\"summary\":\"Flue emits reasoning, tool execution, and LLM turn events separately.\",\"url\":\"https://example.test/flue/reasoning-streams\"}\n\n\nThis is the PREFIX of a turn that was too large to keep. The SUFFIX (recent work) is retained.\n\nSummarize the prefix to provide context for the retained suffix:\n\n## Original Request\n[What did the user ask for in this turn?]\n\n## Early Progress\n- [Key decisions and work done in the prefix]\n\n## Context for Suffix\n- [Information needed to understand the retained recent work]\n\nBe concise. Focus on what's needed to understand the kept suffix.", -│ "type": "text" -│ } -│ ], -│ "role": "user" -│ } -│ ] -│ output: { -│ "content": [ -│ { -│ "text": "## Original Request\nThe user requested to complete an instrumented research flow consisting of three specific steps, involving calling various tools in a sequential manner.\n\n## Early Progress", -│ "textSignature": "{\"v\":1,\"id\":\"msg_01ecf68325f9e202016a184601309081a0878ef4054e950d32\"}", -│ "type": "text" -│ } -│ ], -│ "role": "assistant" -│ } -│ metadata: { -│ "flue.api": "openai-responses", -│ "flue.model": "gpt-4o-mini", -│ "flue.provider": "openai", -│ "flue.session": "main", -│ "flue.stop_reason": "stop", -│ "flue.turn_purpose": "compaction_prefix", -│ "model": "gpt-4o-mini", -│ "provider": "openai" -│ } -│ metrics: { -│ "completion_tokens": 0, -│ "duration_ms": 0, -│ "estimated_cost": 0, -│ "prompt_cache_creation_tokens": 0, -│ "prompt_cached_tokens": 0, -│ "prompt_tokens": 0, -│ "tokens": 0 -│ } -└── flue.prompt [task] - input: [ - { - "content": [ - { - "text": "Complete this instrumented research flow. Call exactly one tool per turn and wait for each tool result before choosing the next tool. Step 1: call lookup with query \"flue instrumentation\". Step 2: use the lookup result id as lookupId and call web_search with query \"Braintrust Flue reasoning stream instrumentation\". Step 3: use the first web_search result url and call summarize_source. After summarize_source returns, reply with exactly PROMPT_DONE and no other text.", - "type": "text" - } - ], - "role": "user" - } - ] - output: "PROMPT_DONE" - metadata: { - "flue.operation": "prompt", - "flue.session": "main", - "flue.workflow_name": "instrumentation", - "provider": "flue", - "scenario": "flue-instrumentation" - } - metrics: { - "duration_ms": 0 - } - ├── flue.turn [llm] - │ input: [ - │ { - │ "content": [ - │ { - │ "text": "Complete this instrumented research flow. Call exactly one tool per turn and wait for each tool result before choosing the next tool. Step 1: call lookup with query \"flue instrumentation\". Step 2: use the lookup result id as lookupId and call web_search with query \"Braintrust Flue reasoning stream instrumentation\". Step 3: use the first web_search result url and call summarize_source. After summarize_source returns, reply with exactly PROMPT_DONE and no other text.", - │ "type": "text" - │ } - │ ], - │ "role": "user" - │ } - │ ] - │ output: { - │ "content": [ - │ { - │ "thinking": "The user wants me to complete a research flow with exactly three steps:\n1. Call lookup with query \"flue instrumentation\"\n2. Call web_search with the lookupId from step 1 and query \"Braintrust Flue reasoning stream instrumentation\"\n3. Call summarize_source with the first URL from step 2\n4. Reply with exactly \"PROMPT_DONE\"\n\nThe user emphasizes I should call exactly one tool per turn and wait for each result. Let me start with step 1.", - │ "thinkingSignature": "EoQFCm4IDhgCKkD+AoKMhpXNPUJ5eANwM28uUCJPCE+AXh0ACw7aXniX+YN04zdJl3hlHMbtAuKhhczLiEeOJtu4KtVpea5u5NFdMhpjbGF1ZGUtc29ubmV0LTQtNS0yMDI1MDkyOTgAQgh0aGlua2luZxIMIOTWCNEtsiI9vF4SGgwPXC861uZx8lpOziEiMLsCukGIluD0U44degzYnHFc+BwXNIEB+yatSHlZE+a5smr9crZcft3OLmsIdXuaFSrDAyQxWd9oqfJdnvza9eWIaOCeNkVCjrTc/wotBwh5Xp8m9k/WKt7P4zzUJBuLQ1f6ZnsdWkL8OiL/2E7Up55z2+UuVWsg1Y90/x27YRFdQjv3RnztYp4bZ5oIHAlTJoCzWiBYEh/Z8c2O+wy6MHo1cp0T57OroB/4HAqRZxrjTOXpKYLmYKXAY5tU8DTd3A0/FfEhg3oPSbOhX+112szFCpm3BRNJYooUUhwSu1hO4H/6r7d/Yp9XIM4tBfzuDx83CC/YetdOFgnYkdliw0xaUGaj4o37rWIRctWHAMdlUc4NDA0Ljc1uirHDXkN8EJwWSoIjPSBTA/p/wgLrvRV5Yy9Z0dMoWOpgYF6t1H/tNAyY8ml1CA7IWyd3jnfTZ1/G3Y0xFDwJDFlb4/QKc67w36SSCa+X9mMoTxD/DYN249lsII+XX3lQ7F1zJ07THOtCyRirCaRhmE/kGwl5tiB1QHrWa9bNHxQxf9i92435JGKN053Fe14AR70MtBSzIU2/zbgvhB1vF5lZt2i46hNxfo4lXR495gEwdwC6+8HJnlEJAJkdeHfZKj+RiwXoiYKPKsSbHi83X8Hh32IifuglNxdlxVQYAQ==", - │ "type": "thinking" - │ }, - │ { - │ "text": "I'll complete this instrumented research flow step by step. Starting with Step 1:", - │ "type": "text" - │ }, - │ { - │ "arguments": { - │ "query": "flue instrumentation" - │ }, - │ "id": "", - │ "name": "lookup", - │ "type": "toolCall" - │ } - │ ], - │ "role": "assistant" - │ } - │ metadata: { - │ "flue.api": "anthropic-messages", - │ "flue.input_mode": "full", - │ "flue.model": "claude-sonnet-4-5-20250929", - │ "flue.provider": "anthropic", - │ "flue.session": "main", - │ "flue.stop_reason": "toolUse", - │ "flue.turn_purpose": "agent", - │ "model": "claude-sonnet-4-5-20250929", - │ "provider": "anthropic" - │ } - │ metrics: { - │ "completion_tokens": 196, - │ "duration_ms": 0, - │ "estimated_cost": 0.0113025, - │ "prompt_cache_creation_tokens": 2222, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 10, - │ "tokens": 2428 - │ } - ├── tool:lookup [tool] - │ input: { - │ "query": "flue instrumentation" - │ } - │ output: { - │ "content": [ - │ { - │ "text": "{\"id\":\"flue-session-2026\",\"query\":\"flue instrumentation\",\"topic\":\"session instrumentation\"}", - │ "type": "text" - │ } - │ ], - │ "details": { - │ "customTool": "lookup" - │ } - │ } - │ metadata: { - │ "flue.session": "main", - │ "flue.tool_name": "lookup", - │ "provider": "flue" - │ } - │ metrics: { - │ "duration_ms": 0 - │ } - ├── flue.turn [llm] - │ input: [ - │ { - │ "content": [ - │ { - │ "thinking": "The user wants me to complete a research flow with exactly three steps:\n1. Call lookup with query \"flue instrumentation\"\n2. Call web_search with the lookupId from step 1 and query \"Braintrust Flue reasoning stream instrumentation\"\n3. Call summarize_source with the first URL from step 2\n4. Reply with exactly \"PROMPT_DONE\"\n\nThe user emphasizes I should call exactly one tool per turn and wait for each result. Let me start with step 1.", - │ "thinkingSignature": "EoQFCm4IDhgCKkD+AoKMhpXNPUJ5eANwM28uUCJPCE+AXh0ACw7aXniX+YN04zdJl3hlHMbtAuKhhczLiEeOJtu4KtVpea5u5NFdMhpjbGF1ZGUtc29ubmV0LTQtNS0yMDI1MDkyOTgAQgh0aGlua2luZxIMIOTWCNEtsiI9vF4SGgwPXC861uZx8lpOziEiMLsCukGIluD0U44degzYnHFc+BwXNIEB+yatSHlZE+a5smr9crZcft3OLmsIdXuaFSrDAyQxWd9oqfJdnvza9eWIaOCeNkVCjrTc/wotBwh5Xp8m9k/WKt7P4zzUJBuLQ1f6ZnsdWkL8OiL/2E7Up55z2+UuVWsg1Y90/x27YRFdQjv3RnztYp4bZ5oIHAlTJoCzWiBYEh/Z8c2O+wy6MHo1cp0T57OroB/4HAqRZxrjTOXpKYLmYKXAY5tU8DTd3A0/FfEhg3oPSbOhX+112szFCpm3BRNJYooUUhwSu1hO4H/6r7d/Yp9XIM4tBfzuDx83CC/YetdOFgnYkdliw0xaUGaj4o37rWIRctWHAMdlUc4NDA0Ljc1uirHDXkN8EJwWSoIjPSBTA/p/wgLrvRV5Yy9Z0dMoWOpgYF6t1H/tNAyY8ml1CA7IWyd3jnfTZ1/G3Y0xFDwJDFlb4/QKc67w36SSCa+X9mMoTxD/DYN249lsII+XX3lQ7F1zJ07THOtCyRirCaRhmE/kGwl5tiB1QHrWa9bNHxQxf9i92435JGKN053Fe14AR70MtBSzIU2/zbgvhB1vF5lZt2i46hNxfo4lXR495gEwdwC6+8HJnlEJAJkdeHfZKj+RiwXoiYKPKsSbHi83X8Hh32IifuglNxdlxVQYAQ==", - │ "type": "thinking" - │ }, - │ { - │ "text": "I'll complete this instrumented research flow step by step. Starting with Step 1:", - │ "type": "text" - │ }, - │ { - │ "arguments": { - │ "query": "flue instrumentation" - │ }, - │ "id": "", - │ "name": "lookup", - │ "type": "toolCall" - │ } - │ ], - │ "role": "assistant" - │ }, - │ { - │ "content": [ - │ { - │ "text": "{\"id\":\"flue-session-2026\",\"query\":\"flue instrumentation\",\"topic\":\"session instrumentation\"}", - │ "type": "text" - │ } - │ ], - │ "isError": false, - │ "role": "toolResult", - │ "toolCallId": "", - │ "toolName": "lookup" - │ } - │ ] - │ output: { - │ "content": [ - │ { - │ "thinking": "Good, I got the lookup result with id \"flue-session-2026\". Now I need to proceed to step 2: call web_search with this lookupId and the query \"Braintrust Flue reasoning stream instrumentation\".", - │ "thinkingSignature": "EpEDCm4IDhgCKkAMahve1kB1j4njp1ZSzO8eCK4bb5Ex7MB6tjdoOXfuUDRfkBqvIGdwVly8GiA8GGYC7Y/RVfREXF6A6OXHTwFyMhpjbGF1ZGUtc29ubmV0LTQtNS0yMDI1MDkyOTgAQgh0aGlua2luZxIMLw+fsa4yo7a7ZSoIGgyaTdvH020n2dQj9bUiMB4OokY29w+uZb4rqDnKDbLXUqUj6U0f1QCGQF2ZBYDDfDJyM5Jv1xY3rQiUUjkhBirQAS4XivjvMnWI5zHBmpI/u/pq4aGdrdb3vUsJ/ne6R1d80JA/INGF8vLkQcZ5xxEK0jU4yRE+ldc2uzavmtoxIKAFD6bqKks1Qv6u+TIZtuW7tUB/7jFxDw93KmJNruMIa8c1J6pPUyBy3xE7QwgmU0kDfIfdnljxLoEBiMwtnxjCPj9lT7HHz4FkrhbnQTV4MvAQohCLjaSbOTd6LdLxwtJPm9mc26ID4eYN1gCCzyAxGXhqLHlzc23PafvQjsZliWnLhaiNMnG0YEo78XkuJQsYAQ==", - │ "type": "thinking" - │ }, - │ { - │ "text": "Now proceeding to Step 2:", - │ "type": "text" - │ }, - │ { - │ "arguments": { - │ "lookupId": "flue-session-2026", - │ "query": "Braintrust Flue reasoning stream instrumentation" - │ }, - │ "id": "", - │ "name": "web_search", - │ "type": "toolCall" - │ } - │ ], - │ "role": "assistant" - │ } - │ metadata: { - │ "flue.api": "anthropic-messages", - │ "flue.input_message_offset": 1, - │ "flue.input_mode": "delta", - │ "flue.model": "claude-sonnet-4-5-20250929", - │ "flue.provider": "anthropic", - │ "flue.session": "main", - │ "flue.stop_reason": "toolUse", - │ "flue.turn_purpose": "agent", - │ "model": "claude-sonnet-4-5-20250929", - │ "provider": "anthropic" - │ } - │ metrics: { - │ "completion_tokens": 158, - │ "duration_ms": 0, - │ "estimated_cost": 0.0039606, - │ "prompt_cache_creation_tokens": 236, - │ "prompt_cached_tokens": 2222, - │ "prompt_tokens": 13, - │ "tokens": 2629 - │ } - ├── tool:web_search [tool] - │ input: { - │ "lookupId": "flue-session-2026", - │ "query": "Braintrust Flue reasoning stream instrumentation" - │ } - │ output: { - │ "content": [ - │ { - │ "text": "{\"lookupId\":\"flue-session-2026\",\"query\":\"Braintrust Flue reasoning stream instrumentation\",\"results\":[{\"title\":\"Flue reasoning stream instrumentation\",\"url\":\"https://example.test/flue/reasoning-streams\"}]}", - │ "type": "text" - │ } - │ ], - │ "details": { - │ "customTool": "web_search" - │ } - │ } - │ metadata: { - │ "flue.session": "main", - │ "flue.tool_name": "web_search", - │ "provider": "flue" - │ } - │ metrics: { - │ "duration_ms": 0 - │ } - ├── flue.turn [llm] - │ input: [ - │ { - │ "content": [ - │ { - │ "thinking": "Good, I got the lookup result with id \"flue-session-2026\". Now I need to proceed to step 2: call web_search with this lookupId and the query \"Braintrust Flue reasoning stream instrumentation\".", - │ "thinkingSignature": "EpEDCm4IDhgCKkAMahve1kB1j4njp1ZSzO8eCK4bb5Ex7MB6tjdoOXfuUDRfkBqvIGdwVly8GiA8GGYC7Y/RVfREXF6A6OXHTwFyMhpjbGF1ZGUtc29ubmV0LTQtNS0yMDI1MDkyOTgAQgh0aGlua2luZxIMLw+fsa4yo7a7ZSoIGgyaTdvH020n2dQj9bUiMB4OokY29w+uZb4rqDnKDbLXUqUj6U0f1QCGQF2ZBYDDfDJyM5Jv1xY3rQiUUjkhBirQAS4XivjvMnWI5zHBmpI/u/pq4aGdrdb3vUsJ/ne6R1d80JA/INGF8vLkQcZ5xxEK0jU4yRE+ldc2uzavmtoxIKAFD6bqKks1Qv6u+TIZtuW7tUB/7jFxDw93KmJNruMIa8c1J6pPUyBy3xE7QwgmU0kDfIfdnljxLoEBiMwtnxjCPj9lT7HHz4FkrhbnQTV4MvAQohCLjaSbOTd6LdLxwtJPm9mc26ID4eYN1gCCzyAxGXhqLHlzc23PafvQjsZliWnLhaiNMnG0YEo78XkuJQsYAQ==", - │ "type": "thinking" - │ }, - │ { - │ "text": "Now proceeding to Step 2:", - │ "type": "text" - │ }, - │ { - │ "arguments": { - │ "lookupId": "flue-session-2026", - │ "query": "Braintrust Flue reasoning stream instrumentation" - │ }, - │ "id": "", - │ "name": "web_search", - │ "type": "toolCall" - │ } - │ ], - │ "role": "assistant" - │ }, - │ { - │ "content": [ - │ { - │ "text": "{\"lookupId\":\"flue-session-2026\",\"query\":\"Braintrust Flue reasoning stream instrumentation\",\"results\":[{\"title\":\"Flue reasoning stream instrumentation\",\"url\":\"https://example.test/flue/reasoning-streams\"}]}", - │ "type": "text" - │ } - │ ], - │ "isError": false, - │ "role": "toolResult", - │ "toolCallId": "", - │ "toolName": "web_search" - │ } - │ ] - │ output: { - │ "content": [ - │ { - │ "thinking": "Perfect! I got the web_search results with one result. The first (and only) result has the URL \"https://example.test/flue/reasoning-streams\". Now I need to proceed to step 3: call summarize_source with this URL.", - │ "thinkingSignature": "EqQDCm4IDhgCKkAXv7EDAhWMcrBz11bUd5qxlzyy03CAGQfRgLHDE0uKzLATh1o2sedd8GxLwwQqV0CmLRfNxsdK/ZDp3Lks9iysMhpjbGF1ZGUtc29ubmV0LTQtNS0yMDI1MDkyOTgAQgh0aGlua2luZxIMXDMniOy5Dy0KI8x7GgzgHs7BYJMs6/N0ur8iMPe0SwnHhP39OjEu7P8ASwK9ffwa3VP67QOlPzOstRRdEZNd0mq+BVjpmBXB+QcAzCrjAeAh/UO/d7SfFmXjqwA3//thoL/TTlGa6W+fdOCFRikn65JWT4ZdVFetfqD+9/QCP/nXEwMsLnTBAxC4EGryrgs2YT+OWwmEJ7rHKFeDs5djhwSb2J2ZJO1kRvD9s2vCCDkImygYBSIVgx1Y0CZJHhzn2Vf39v0Fbth2wDZa+OqxivV9NmDZEjmgcM+tTRx1CCnaww8LZEJQ68v/dUJy7hhD4MrNcQ62evJYPfT3+4EKKWJ/7HI7KXz91//+YAkph63XstbJMFJ3AldAFiq3ecauXt5r42N5gmCmwZX+sI48ENLGGAE=", - │ "type": "thinking" - │ }, - │ { - │ "text": "Now proceeding to Step 3:", - │ "type": "text" - │ }, - │ { - │ "arguments": { - │ "url": "https://example.test/flue/reasoning-streams" - │ }, - │ "id": "", - │ "name": "summarize_source", - │ "type": "toolCall" - │ } - │ ], - │ "role": "assistant" - │ } - │ metadata: { - │ "flue.api": "anthropic-messages", - │ "flue.input_message_offset": 3, - │ "flue.input_mode": "delta", - │ "flue.model": "claude-sonnet-4-5-20250929", - │ "flue.provider": "anthropic", - │ "flue.session": "main", - │ "flue.stop_reason": "toolUse", - │ "flue.turn_purpose": "agent", - │ "model": "claude-sonnet-4-5-20250929", - │ "provider": "anthropic" - │ } - │ metrics: { - │ "completion_tokens": 138, - │ "duration_ms": 0, - │ "estimated_cost": 0.0037239000000000005, - │ "prompt_cache_creation_tokens": 234, - │ "prompt_cached_tokens": 2458, - │ "prompt_tokens": 13, - │ "tokens": 2843 - │ } - ├── tool:summarize_source [tool] - │ input: { - │ "url": "https://example.test/flue/reasoning-streams" - │ } - │ output: { - │ "content": [ - │ { - │ "text": "{\"summary\":\"Flue emits reasoning, tool execution, and LLM turn events separately.\",\"url\":\"https://example.test/flue/reasoning-streams\"}", - │ "type": "text" - │ } - │ ], - │ "details": { - │ "customTool": "summarize_source" - │ } - │ } - │ metadata: { - │ "flue.session": "main", - │ "flue.tool_name": "summarize_source", - │ "provider": "flue" - │ } - │ metrics: { - │ "duration_ms": 0 - │ } - └── flue.turn [llm] - input: [ - { - "content": [ - { - "thinking": "Perfect! I got the web_search results with one result. The first (and only) result has the URL \"https://example.test/flue/reasoning-streams\". Now I need to proceed to step 3: call summarize_source with this URL.", - "thinkingSignature": "EqQDCm4IDhgCKkAXv7EDAhWMcrBz11bUd5qxlzyy03CAGQfRgLHDE0uKzLATh1o2sedd8GxLwwQqV0CmLRfNxsdK/ZDp3Lks9iysMhpjbGF1ZGUtc29ubmV0LTQtNS0yMDI1MDkyOTgAQgh0aGlua2luZxIMXDMniOy5Dy0KI8x7GgzgHs7BYJMs6/N0ur8iMPe0SwnHhP39OjEu7P8ASwK9ffwa3VP67QOlPzOstRRdEZNd0mq+BVjpmBXB+QcAzCrjAeAh/UO/d7SfFmXjqwA3//thoL/TTlGa6W+fdOCFRikn65JWT4ZdVFetfqD+9/QCP/nXEwMsLnTBAxC4EGryrgs2YT+OWwmEJ7rHKFeDs5djhwSb2J2ZJO1kRvD9s2vCCDkImygYBSIVgx1Y0CZJHhzn2Vf39v0Fbth2wDZa+OqxivV9NmDZEjmgcM+tTRx1CCnaww8LZEJQ68v/dUJy7hhD4MrNcQ62evJYPfT3+4EKKWJ/7HI7KXz91//+YAkph63XstbJMFJ3AldAFiq3ecauXt5r42N5gmCmwZX+sI48ENLGGAE=", - "type": "thinking" - }, - { - "text": "Now proceeding to Step 3:", - "type": "text" - }, - { - "arguments": { - "url": "https://example.test/flue/reasoning-streams" - }, - "id": "", - "name": "summarize_source", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "{\"summary\":\"Flue emits reasoning, tool execution, and LLM turn events separately.\",\"url\":\"https://example.test/flue/reasoning-streams\"}", - "type": "text" - } - ], - "isError": false, - "role": "toolResult", - "toolCallId": "", - "toolName": "summarize_source" - } - ] - output: { - "content": [ - { - "thinking": "Perfect! The summarize_source call has completed successfully. According to the instructions, I should now reply with exactly \"PROMPT_DONE\" and no other text.", - "thinkingSignature": "Eu8CCm4IDhgCKkAn2QOIIlYfvcs6ynsmhCslHDtBsrTA4CfmimK7zyz6+k6dZldV2JBtEBET41It0gFcGEKC0n0mAhaYVuLutpDDMhpjbGF1ZGUtc29ubmV0LTQtNS0yMDI1MDkyOTgAQgh0aGlua2luZxIM1xfLptxSF8rKzou8GgwRLuVmfiBYOT4CZIsiMAYz8TpvchMAWodP+T9uZ+dJ47MVdNv+p86nZXmT5d5EWlCroiAOFHHzJH2o0UjJQSquAX6wu+jIC8/DXwurGIIf2ToldPuJWF5HGj8kHMnwh777XS1NUFl1irM24rvP7kTAv3Dp8YEwb8t3g1EsicY7j+C5DzgwxVsPYCVOCgdMqQjtsqpL7AX5KQ9+hR1gK1DQjshv4Gze3hw/+C+9iptd3RlDIuwO0cm8NY+sVqeWVh3TtQujQlVt1rW/HfWjclrtT0LZSGwc6RypmfMMQi6lncIk/EPgy0XbS1plKW0tLBgB", - "type": "thinking" - }, - { - "text": "PROMPT_DONE", - "type": "text" - } - ], - "role": "assistant" - } - metadata: { - "flue.api": "anthropic-messages", - "flue.input_message_offset": 5, - "flue.input_mode": "delta", - "flue.model": "claude-sonnet-4-5-20250929", - "flue.provider": "anthropic", - "flue.session": "main", - "flue.stop_reason": "stop", - "flue.turn_purpose": "agent", - "model": "claude-sonnet-4-5-20250929", - "provider": "anthropic" - } - metrics: { - "completion_tokens": 51, - "duration_ms": 0, - "estimated_cost": 0.0023391, - "prompt_cache_creation_tokens": 194, - "prompt_cached_tokens": 2692, - "prompt_tokens": 13, - "tokens": 2950 - } diff --git a/e2e/scenarios/flue-instrumentation/__snapshots__/flue-v0-8-0-cli.span-tree.json b/e2e/scenarios/flue-instrumentation/__snapshots__/flue-v0-8-0-cli.span-tree.json deleted file mode 100644 index dd92abe29..000000000 --- a/e2e/scenarios/flue-instrumentation/__snapshots__/flue-v0-8-0-cli.span-tree.json +++ /dev/null @@ -1,1448 +0,0 @@ -{ - "span_tree": [ - { - "name": "workflow:instrumentation", - "type": "task", - "children": [ - { - "name": "flue.skill", - "type": "task", - "children": [ - { - "name": "flue.turn", - "type": "llm", - "children": [], - "input": [ - { - "content": [ - { - "text": "Run the skill named \"e2e-flue-skill\".\n\nArguments:\n{\n \"marker\": \"SKILL_DONE\"\n}", - "type": "text" - } - ], - "role": "user" - } - ], - "output": { - "content": [ - { - "text": "I'll run the e2e-flue-skill. Let me first read the skill file to understand its instructions.", - "type": "text" - }, - { - "arguments": { - "path": ".agents/e2e-flue-skill.md" - }, - "id": "", - "name": "read", - "type": "toolCall" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "anthropic-messages", - "flue.model": "claude-sonnet-4-5-20250929", - "flue.provider": "anthropic", - "flue.session": "skill", - "flue.stop_reason": "toolUse", - "flue.turn_purpose": "agent", - "model": "claude-sonnet-4-5-20250929", - "provider": "anthropic" - }, - "metrics": { - "completion_tokens": 89, - "duration_ms": 0, - "estimated_cost": 0.0041862, - "prompt_cache_creation_tokens": 670, - "prompt_cached_tokens": 1099, - "prompt_tokens": 3, - "tokens": 1861 - } - }, - { - "name": "tool:read", - "type": "tool", - "children": [], - "input": { - "path": ".agents/e2e-flue-skill.md" - }, - "output": { - "content": [ - { - "text": "ENOENT: no such file or directory, open '/e2e/scenarios/flue-instrumentation/.agents/e2e-flue-skill.md'", - "type": "text" - } - ], - "details": {} - }, - "metadata": { - "flue.session": "skill", - "flue.tool_name": "read", - "provider": "flue" - }, - "metrics": { - "duration_ms": 0 - }, - "error": "{\"content\":[{\"type\":\"text\",\"text\":\"ENOENT: no such file or directory, open '/e2e/scenarios/flue-instrumentation/.agents/e2e-flue-skill.md'\"}],\"details\":{}}" - }, - { - "name": "flue.turn", - "type": "llm", - "children": [], - "input": [ - { - "content": [ - { - "text": "Run the skill named \"e2e-flue-skill\".\n\nArguments:\n{\n \"marker\": \"SKILL_DONE\"\n}", - "type": "text" - } - ], - "role": "user" - }, - { - "content": [ - { - "text": "I'll run the e2e-flue-skill. Let me first read the skill file to understand its instructions.", - "type": "text" - }, - { - "arguments": { - "path": ".agents/e2e-flue-skill.md" - }, - "id": "", - "name": "read", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "ENOENT: no such file or directory, open '/e2e/scenarios/flue-instrumentation/.agents/e2e-flue-skill.md'", - "type": "text" - } - ], - "isError": true, - "role": "toolResult", - "toolCallId": "", - "toolName": "read" - } - ], - "output": { - "content": [ - { - "arguments": { - "path": ".agents" - }, - "id": "", - "name": "read", - "type": "toolCall" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "anthropic-messages", - "flue.model": "claude-sonnet-4-5-20250929", - "flue.provider": "anthropic", - "flue.session": "skill", - "flue.stop_reason": "toolUse", - "flue.turn_purpose": "agent", - "model": "claude-sonnet-4-5-20250929", - "provider": "anthropic" - }, - "metrics": { - "completion_tokens": 52, - "duration_ms": 0, - "estimated_cost": 0.00201795, - "prompt_cache_creation_tokens": 183, - "prompt_cached_tokens": 1769, - "prompt_tokens": 7, - "tokens": 2011 - } - }, - { - "name": "tool:read", - "type": "tool", - "children": [], - "input": { - "path": ".agents" - }, - "output": { - "content": [ - { - "text": "skills", - "type": "text" - } - ], - "details": { - "entries": 1, - "isDirectory": true, - "path": ".agents" - } - }, - "metadata": { - "flue.session": "skill", - "flue.tool_name": "read", - "provider": "flue" - }, - "metrics": { - "duration_ms": 0 - } - }, - { - "name": "flue.turn", - "type": "llm", - "children": [], - "input": [ - { - "content": [ - { - "text": "Run the skill named \"e2e-flue-skill\".\n\nArguments:\n{\n \"marker\": \"SKILL_DONE\"\n}", - "type": "text" - } - ], - "role": "user" - }, - { - "content": [ - { - "text": "I'll run the e2e-flue-skill. Let me first read the skill file to understand its instructions.", - "type": "text" - }, - { - "arguments": { - "path": ".agents/e2e-flue-skill.md" - }, - "id": "", - "name": "read", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "ENOENT: no such file or directory, open '/e2e/scenarios/flue-instrumentation/.agents/e2e-flue-skill.md'", - "type": "text" - } - ], - "isError": true, - "role": "toolResult", - "toolCallId": "", - "toolName": "read" - }, - { - "content": [ - { - "arguments": { - "path": ".agents" - }, - "id": "", - "name": "read", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "skills", - "type": "text" - } - ], - "isError": false, - "role": "toolResult", - "toolCallId": "", - "toolName": "read" - } - ], - "output": { - "content": [ - { - "arguments": { - "path": ".agents/skills" - }, - "id": "", - "name": "read", - "type": "toolCall" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "anthropic-messages", - "flue.model": "claude-sonnet-4-5-20250929", - "flue.provider": "anthropic", - "flue.session": "skill", - "flue.stop_reason": "toolUse", - "flue.turn_purpose": "agent", - "model": "claude-sonnet-4-5-20250929", - "provider": "anthropic" - }, - "metrics": { - "completion_tokens": 54, - "duration_ms": 0, - "estimated_cost": 0.0016611, - "prompt_cache_creation_tokens": 66, - "prompt_cached_tokens": 1952, - "prompt_tokens": 6, - "tokens": 2078 - } - }, - { - "name": "tool:read", - "type": "tool", - "children": [], - "input": { - "path": ".agents/skills" - }, - "output": { - "content": [ - { - "text": "e2e-flue-skill", - "type": "text" - } - ], - "details": { - "entries": 1, - "isDirectory": true, - "path": ".agents/skills" - } - }, - "metadata": { - "flue.session": "skill", - "flue.tool_name": "read", - "provider": "flue" - }, - "metrics": { - "duration_ms": 0 - } - }, - { - "name": "flue.turn", - "type": "llm", - "children": [], - "input": [ - { - "content": [ - { - "text": "Run the skill named \"e2e-flue-skill\".\n\nArguments:\n{\n \"marker\": \"SKILL_DONE\"\n}", - "type": "text" - } - ], - "role": "user" - }, - { - "content": [ - { - "text": "I'll run the e2e-flue-skill. Let me first read the skill file to understand its instructions.", - "type": "text" - }, - { - "arguments": { - "path": ".agents/e2e-flue-skill.md" - }, - "id": "", - "name": "read", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "ENOENT: no such file or directory, open '/e2e/scenarios/flue-instrumentation/.agents/e2e-flue-skill.md'", - "type": "text" - } - ], - "isError": true, - "role": "toolResult", - "toolCallId": "", - "toolName": "read" - }, - { - "content": [ - { - "arguments": { - "path": ".agents" - }, - "id": "", - "name": "read", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "skills", - "type": "text" - } - ], - "isError": false, - "role": "toolResult", - "toolCallId": "", - "toolName": "read" - }, - { - "content": [ - { - "arguments": { - "path": ".agents/skills" - }, - "id": "", - "name": "read", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "e2e-flue-skill", - "type": "text" - } - ], - "isError": false, - "role": "toolResult", - "toolCallId": "", - "toolName": "read" - } - ], - "output": { - "content": [ - { - "arguments": { - "path": ".agents/skills/e2e-flue-skill" - }, - "id": "", - "name": "read", - "type": "toolCall" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "anthropic-messages", - "flue.model": "claude-sonnet-4-5-20250929", - "flue.provider": "anthropic", - "flue.session": "skill", - "flue.stop_reason": "toolUse", - "flue.turn_purpose": "agent", - "model": "claude-sonnet-4-5-20250929", - "provider": "anthropic" - }, - "metrics": { - "completion_tokens": 63, - "duration_ms": 0, - "estimated_cost": 0.0018459, - "prompt_cache_creation_tokens": 74, - "prompt_cached_tokens": 2018, - "prompt_tokens": 6, - "tokens": 2161 - } - }, - { - "name": "tool:read", - "type": "tool", - "children": [], - "input": { - "path": ".agents/skills/e2e-flue-skill" - }, - "output": { - "content": [ - { - "text": "SKILL.md", - "type": "text" - } - ], - "details": { - "entries": 1, - "isDirectory": true, - "path": ".agents/skills/e2e-flue-skill" - } - }, - "metadata": { - "flue.session": "skill", - "flue.tool_name": "read", - "provider": "flue" - }, - "metrics": { - "duration_ms": 0 - } - }, - { - "name": "flue.turn", - "type": "llm", - "children": [], - "input": [ - { - "content": [ - { - "text": "Run the skill named \"e2e-flue-skill\".\n\nArguments:\n{\n \"marker\": \"SKILL_DONE\"\n}", - "type": "text" - } - ], - "role": "user" - }, - { - "content": [ - { - "text": "I'll run the e2e-flue-skill. Let me first read the skill file to understand its instructions.", - "type": "text" - }, - { - "arguments": { - "path": ".agents/e2e-flue-skill.md" - }, - "id": "", - "name": "read", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "ENOENT: no such file or directory, open '/e2e/scenarios/flue-instrumentation/.agents/e2e-flue-skill.md'", - "type": "text" - } - ], - "isError": true, - "role": "toolResult", - "toolCallId": "", - "toolName": "read" - }, - { - "content": [ - { - "arguments": { - "path": ".agents" - }, - "id": "", - "name": "read", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "skills", - "type": "text" - } - ], - "isError": false, - "role": "toolResult", - "toolCallId": "", - "toolName": "read" - }, - { - "content": [ - { - "arguments": { - "path": ".agents/skills" - }, - "id": "", - "name": "read", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "e2e-flue-skill", - "type": "text" - } - ], - "isError": false, - "role": "toolResult", - "toolCallId": "", - "toolName": "read" - }, - { - "content": [ - { - "arguments": { - "path": ".agents/skills/e2e-flue-skill" - }, - "id": "", - "name": "read", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "SKILL.md", - "type": "text" - } - ], - "isError": false, - "role": "toolResult", - "toolCallId": "", - "toolName": "read" - } - ], - "output": { - "content": [ - { - "arguments": { - "path": ".agents/skills/e2e-flue-skill/SKILL.md" - }, - "id": "", - "name": "read", - "type": "toolCall" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "anthropic-messages", - "flue.model": "claude-sonnet-4-5-20250929", - "flue.provider": "anthropic", - "flue.session": "skill", - "flue.stop_reason": "toolUse", - "flue.turn_purpose": "agent", - "model": "claude-sonnet-4-5-20250929", - "provider": "anthropic" - }, - "metrics": { - "completion_tokens": 68, - "duration_ms": 0, - "estimated_cost": 0.0019618500000000002, - "prompt_cache_creation_tokens": 79, - "prompt_cached_tokens": 2092, - "prompt_tokens": 6, - "tokens": 2245 - } - }, - { - "name": "tool:read", - "type": "tool", - "children": [], - "input": { - "path": ".agents/skills/e2e-flue-skill/SKILL.md" - }, - "output": { - "content": [ - { - "text": "---\nname: e2e-flue-skill\ndescription: Deterministic skill used by the Braintrust Flue instrumentation e2e test.\n---\n\nReturn the marker from the provided args exactly once. Output no other text.\n", - "type": "text" - } - ], - "details": { - "lines": 7, - "path": ".agents/skills/e2e-flue-skill/SKILL.md" - } - }, - "metadata": { - "flue.session": "skill", - "flue.tool_name": "read", - "provider": "flue" - }, - "metrics": { - "duration_ms": 0 - } - }, - { - "name": "flue.turn", - "type": "llm", - "children": [], - "input": [ - { - "content": [ - { - "text": "Run the skill named \"e2e-flue-skill\".\n\nArguments:\n{\n \"marker\": \"SKILL_DONE\"\n}", - "type": "text" - } - ], - "role": "user" - }, - { - "content": [ - { - "text": "I'll run the e2e-flue-skill. Let me first read the skill file to understand its instructions.", - "type": "text" - }, - { - "arguments": { - "path": ".agents/e2e-flue-skill.md" - }, - "id": "", - "name": "read", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "ENOENT: no such file or directory, open '/e2e/scenarios/flue-instrumentation/.agents/e2e-flue-skill.md'", - "type": "text" - } - ], - "isError": true, - "role": "toolResult", - "toolCallId": "", - "toolName": "read" - }, - { - "content": [ - { - "arguments": { - "path": ".agents" - }, - "id": "", - "name": "read", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "skills", - "type": "text" - } - ], - "isError": false, - "role": "toolResult", - "toolCallId": "", - "toolName": "read" - }, - { - "content": [ - { - "arguments": { - "path": ".agents/skills" - }, - "id": "", - "name": "read", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "e2e-flue-skill", - "type": "text" - } - ], - "isError": false, - "role": "toolResult", - "toolCallId": "", - "toolName": "read" - }, - { - "content": [ - { - "arguments": { - "path": ".agents/skills/e2e-flue-skill" - }, - "id": "", - "name": "read", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "SKILL.md", - "type": "text" - } - ], - "isError": false, - "role": "toolResult", - "toolCallId": "", - "toolName": "read" - }, - { - "content": [ - { - "arguments": { - "path": ".agents/skills/e2e-flue-skill/SKILL.md" - }, - "id": "", - "name": "read", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "---\nname: e2e-flue-skill\ndescription: Deterministic skill used by the Braintrust Flue instrumentation e2e test.\n---\n\nReturn the marker from the provided args exactly once. Output no other text.\n", - "type": "text" - } - ], - "isError": false, - "role": "toolResult", - "toolCallId": "", - "toolName": "read" - } - ], - "output": { - "content": [ - { - "text": "SKILL_DONE", - "type": "text" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "anthropic-messages", - "flue.model": "claude-sonnet-4-5-20250929", - "flue.provider": "anthropic", - "flue.session": "skill", - "flue.stop_reason": "stop", - "flue.turn_purpose": "agent", - "model": "claude-sonnet-4-5-20250929", - "provider": "anthropic" - }, - "metrics": { - "completion_tokens": 8, - "duration_ms": 0, - "estimated_cost": 0.00129555, - "prompt_cache_creation_tokens": 135, - "prompt_cached_tokens": 2171, - "prompt_tokens": 6, - "tokens": 2320 - } - } - ], - "input": [ - { - "content": [ - { - "text": "Run the skill named \"e2e-flue-skill\".\n\nArguments:\n{\n \"marker\": \"SKILL_DONE\"\n}", - "type": "text" - } - ], - "role": "user" - } - ], - "output": "SKILL_DONE", - "metadata": { - "flue.operation": "skill", - "flue.session": "skill", - "flue.workflow_name": "instrumentation", - "provider": "flue", - "scenario": "flue-instrumentation" - }, - "metrics": { - "duration_ms": 0 - } - }, - { - "name": "flue.task", - "type": "task", - "children": [ - { - "name": "flue.turn", - "type": "llm", - "children": [], - "input": [ - { - "content": [ - { - "text": "Reply with exactly TASK_DONE and no other text.", - "type": "text" - } - ], - "role": "user" - } - ], - "output": { - "content": [ - { - "text": "TASK_DONE", - "textSignature": "{\"v\":1,\"id\":\"msg_09b32b2821e9a4d8016a184600019081929d34bb7d4c6a246f\"}", - "type": "text" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "openai-responses", - "flue.input_mode": "full", - "flue.model": "gpt-4o-mini", - "flue.provider": "openai", - "flue.session": "task:task:", - "flue.stop_reason": "stop", - "flue.turn_purpose": "agent", - "model": "gpt-4o-mini", - "provider": "openai" - }, - "metrics": { - "completion_tokens": 4, - "duration_ms": 0, - "estimated_cost": 0.00012299999999999998, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 804, - "tokens": 808 - } - } - ], - "input": "Reply with exactly TASK_DONE and no other text.", - "output": "TASK_DONE", - "metadata": { - "flue.session": "task", - "provider": "flue" - }, - "metrics": { - "duration_ms": 0 - } - }, - { - "name": "flue.compact", - "type": "task", - "children": [ - { - "name": "compaction:manual", - "type": "task", - "children": [ - { - "name": "flue.turn", - "type": "llm", - "children": [], - "input": [ - { - "content": [ - { - "text": "\n[User]: Complete this instrumented research flow. Call exactly one tool per turn and wait for each tool result before choosing the next tool. Step 1: call lookup with query \"flue instrumentation\". Step 2: use the lookup result id as lookupId and call web_search with query \"Braintrust Flue reasoning stream instrumentation\". Step 3: use the first web_search result url and call summarize_source. After summarize_source returns, reply with exactly PROMPT_DONE and no other text.\n\n[Assistant thinking]: The user wants me to complete a research flow with exactly three steps:\n1. Call lookup with query \"flue instrumentation\"\n2. Call web_search with the lookupId from step 1 and query \"Braintrust Flue reasoning stream instrumentation\"\n3. Call summarize_source with the first URL from step 2\n4. Reply with exactly \"PROMPT_DONE\"\n\nThe user emphasizes I should call exactly one tool per turn and wait for each result. Let me start with step 1.\n\n[Assistant]: I'll complete this instrumented research flow step by step. Starting with Step 1:\n\n[Assistant tool calls]: lookup(query=\"flue instrumentation\")\n\n[Tool result]: {\"id\":\"flue-session-2026\",\"query\":\"flue instrumentation\",\"topic\":\"session instrumentation\"}\n\n[Assistant thinking]: Good, I got the lookup result with id \"flue-session-2026\". Now I need to proceed to step 2: call web_search with this lookupId and the query \"Braintrust Flue reasoning stream instrumentation\".\n\n[Assistant]: Now proceeding to Step 2:\n\n[Assistant tool calls]: web_search(lookupId=\"flue-session-2026\", query=\"Braintrust Flue reasoning stream instrumentation\")\n\n[Tool result]: {\"lookupId\":\"flue-session-2026\",\"query\":\"Braintrust Flue reasoning stream instrumentation\",\"results\":[{\"title\":\"Flue reasoning stream instrumentation\",\"url\":\"https://example.test/flue/reasoning-streams\"}]}\n\n[Assistant thinking]: Perfect! I got the web_search results with one result. The first (and only) result has the URL \"https://example.test/flue/reasoning-streams\". Now I need to proceed to step 3: call summarize_source with this URL.\n\n[Assistant]: Now proceeding to Step 3:\n\n[Assistant tool calls]: summarize_source(url=\"https://example.test/flue/reasoning-streams\")\n\n[Tool result]: {\"summary\":\"Flue emits reasoning, tool execution, and LLM turn events separately.\",\"url\":\"https://example.test/flue/reasoning-streams\"}\n\n\nThis is the PREFIX of a turn that was too large to keep. The SUFFIX (recent work) is retained.\n\nSummarize the prefix to provide context for the retained suffix:\n\n## Original Request\n[What did the user ask for in this turn?]\n\n## Early Progress\n- [Key decisions and work done in the prefix]\n\n## Context for Suffix\n- [Information needed to understand the retained recent work]\n\nBe concise. Focus on what's needed to understand the kept suffix.", - "type": "text" - } - ], - "role": "user" - } - ], - "output": { - "content": [ - { - "text": "## Original Request\nThe user requested to complete an instrumented research flow consisting of three specific steps, involving calling various tools in a sequential manner.\n\n## Early Progress", - "textSignature": "{\"v\":1,\"id\":\"msg_01ecf68325f9e202016a184601309081a0878ef4054e950d32\"}", - "type": "text" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "openai-responses", - "flue.model": "gpt-4o-mini", - "flue.provider": "openai", - "flue.session": "main", - "flue.stop_reason": "stop", - "flue.turn_purpose": "compaction_prefix", - "model": "gpt-4o-mini", - "provider": "openai" - }, - "metrics": { - "completion_tokens": 0, - "duration_ms": 0, - "estimated_cost": 0, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 0, - "tokens": 0 - } - } - ], - "input": { - "estimatedTokens": 2950, - "reason": "manual" - }, - "output": { - "messagesAfter": 2, - "messagesBefore": 8 - }, - "metadata": { - "flue.compaction_reason": "manual", - "flue.session": "main", - "provider": "flue" - }, - "metrics": { - "duration_ms": 0, - "messages_after": 2, - "messages_before": 8 - } - } - ], - "input": { - "estimatedTokens": 2950, - "reason": "manual" - }, - "output": { - "completed": true - }, - "metadata": { - "flue.operation": "compact", - "flue.session": "main", - "flue.workflow_name": "instrumentation", - "provider": "flue", - "scenario": "flue-instrumentation" - }, - "metrics": { - "duration_ms": 0 - } - } - ], - "input": { - "metadata": { - "scenario": "flue-instrumentation", - "testRunId": "" - }, - "scenario": "flue-instrumentation" - }, - "output": { - "scenario": "flue-instrumentation", - "status": "done" - }, - "metadata": { - "flue.workflow_name": "instrumentation", - "provider": "flue", - "scenario": "flue-instrumentation" - }, - "metrics": { - "duration_ms": 0 - } - }, - { - "name": "flue.prompt", - "type": "task", - "children": [ - { - "name": "flue.turn", - "type": "llm", - "children": [], - "input": [ - { - "content": [ - { - "text": "Complete this instrumented research flow. Call exactly one tool per turn and wait for each tool result before choosing the next tool. Step 1: call lookup with query \"flue instrumentation\". Step 2: use the lookup result id as lookupId and call web_search with query \"Braintrust Flue reasoning stream instrumentation\". Step 3: use the first web_search result url and call summarize_source. After summarize_source returns, reply with exactly PROMPT_DONE and no other text.", - "type": "text" - } - ], - "role": "user" - } - ], - "output": { - "content": [ - { - "thinking": "The user wants me to complete a research flow with exactly three steps:\n1. Call lookup with query \"flue instrumentation\"\n2. Call web_search with the lookupId from step 1 and query \"Braintrust Flue reasoning stream instrumentation\"\n3. Call summarize_source with the first URL from step 2\n4. Reply with exactly \"PROMPT_DONE\"\n\nThe user emphasizes I should call exactly one tool per turn and wait for each result. Let me start with step 1.", - "thinkingSignature": "EoQFCm4IDhgCKkD+AoKMhpXNPUJ5eANwM28uUCJPCE+AXh0ACw7aXniX+YN04zdJl3hlHMbtAuKhhczLiEeOJtu4KtVpea5u5NFdMhpjbGF1ZGUtc29ubmV0LTQtNS0yMDI1MDkyOTgAQgh0aGlua2luZxIMIOTWCNEtsiI9vF4SGgwPXC861uZx8lpOziEiMLsCukGIluD0U44degzYnHFc+BwXNIEB+yatSHlZE+a5smr9crZcft3OLmsIdXuaFSrDAyQxWd9oqfJdnvza9eWIaOCeNkVCjrTc/wotBwh5Xp8m9k/WKt7P4zzUJBuLQ1f6ZnsdWkL8OiL/2E7Up55z2+UuVWsg1Y90/x27YRFdQjv3RnztYp4bZ5oIHAlTJoCzWiBYEh/Z8c2O+wy6MHo1cp0T57OroB/4HAqRZxrjTOXpKYLmYKXAY5tU8DTd3A0/FfEhg3oPSbOhX+112szFCpm3BRNJYooUUhwSu1hO4H/6r7d/Yp9XIM4tBfzuDx83CC/YetdOFgnYkdliw0xaUGaj4o37rWIRctWHAMdlUc4NDA0Ljc1uirHDXkN8EJwWSoIjPSBTA/p/wgLrvRV5Yy9Z0dMoWOpgYF6t1H/tNAyY8ml1CA7IWyd3jnfTZ1/G3Y0xFDwJDFlb4/QKc67w36SSCa+X9mMoTxD/DYN249lsII+XX3lQ7F1zJ07THOtCyRirCaRhmE/kGwl5tiB1QHrWa9bNHxQxf9i92435JGKN053Fe14AR70MtBSzIU2/zbgvhB1vF5lZt2i46hNxfo4lXR495gEwdwC6+8HJnlEJAJkdeHfZKj+RiwXoiYKPKsSbHi83X8Hh32IifuglNxdlxVQYAQ==", - "type": "thinking" - }, - { - "text": "I'll complete this instrumented research flow step by step. Starting with Step 1:", - "type": "text" - }, - { - "arguments": { - "query": "flue instrumentation" - }, - "id": "", - "name": "lookup", - "type": "toolCall" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "anthropic-messages", - "flue.input_mode": "full", - "flue.model": "claude-sonnet-4-5-20250929", - "flue.provider": "anthropic", - "flue.session": "main", - "flue.stop_reason": "toolUse", - "flue.turn_purpose": "agent", - "model": "claude-sonnet-4-5-20250929", - "provider": "anthropic" - }, - "metrics": { - "completion_tokens": 196, - "duration_ms": 0, - "estimated_cost": 0.0113025, - "prompt_cache_creation_tokens": 2222, - "prompt_cached_tokens": 0, - "prompt_tokens": 10, - "tokens": 2428 - } - }, - { - "name": "tool:lookup", - "type": "tool", - "children": [], - "input": { - "query": "flue instrumentation" - }, - "output": { - "content": [ - { - "text": "{\"id\":\"flue-session-2026\",\"query\":\"flue instrumentation\",\"topic\":\"session instrumentation\"}", - "type": "text" - } - ], - "details": { - "customTool": "lookup" - } - }, - "metadata": { - "flue.session": "main", - "flue.tool_name": "lookup", - "provider": "flue" - }, - "metrics": { - "duration_ms": 0 - } - }, - { - "name": "flue.turn", - "type": "llm", - "children": [], - "input": [ - { - "content": [ - { - "thinking": "The user wants me to complete a research flow with exactly three steps:\n1. Call lookup with query \"flue instrumentation\"\n2. Call web_search with the lookupId from step 1 and query \"Braintrust Flue reasoning stream instrumentation\"\n3. Call summarize_source with the first URL from step 2\n4. Reply with exactly \"PROMPT_DONE\"\n\nThe user emphasizes I should call exactly one tool per turn and wait for each result. Let me start with step 1.", - "thinkingSignature": "EoQFCm4IDhgCKkD+AoKMhpXNPUJ5eANwM28uUCJPCE+AXh0ACw7aXniX+YN04zdJl3hlHMbtAuKhhczLiEeOJtu4KtVpea5u5NFdMhpjbGF1ZGUtc29ubmV0LTQtNS0yMDI1MDkyOTgAQgh0aGlua2luZxIMIOTWCNEtsiI9vF4SGgwPXC861uZx8lpOziEiMLsCukGIluD0U44degzYnHFc+BwXNIEB+yatSHlZE+a5smr9crZcft3OLmsIdXuaFSrDAyQxWd9oqfJdnvza9eWIaOCeNkVCjrTc/wotBwh5Xp8m9k/WKt7P4zzUJBuLQ1f6ZnsdWkL8OiL/2E7Up55z2+UuVWsg1Y90/x27YRFdQjv3RnztYp4bZ5oIHAlTJoCzWiBYEh/Z8c2O+wy6MHo1cp0T57OroB/4HAqRZxrjTOXpKYLmYKXAY5tU8DTd3A0/FfEhg3oPSbOhX+112szFCpm3BRNJYooUUhwSu1hO4H/6r7d/Yp9XIM4tBfzuDx83CC/YetdOFgnYkdliw0xaUGaj4o37rWIRctWHAMdlUc4NDA0Ljc1uirHDXkN8EJwWSoIjPSBTA/p/wgLrvRV5Yy9Z0dMoWOpgYF6t1H/tNAyY8ml1CA7IWyd3jnfTZ1/G3Y0xFDwJDFlb4/QKc67w36SSCa+X9mMoTxD/DYN249lsII+XX3lQ7F1zJ07THOtCyRirCaRhmE/kGwl5tiB1QHrWa9bNHxQxf9i92435JGKN053Fe14AR70MtBSzIU2/zbgvhB1vF5lZt2i46hNxfo4lXR495gEwdwC6+8HJnlEJAJkdeHfZKj+RiwXoiYKPKsSbHi83X8Hh32IifuglNxdlxVQYAQ==", - "type": "thinking" - }, - { - "text": "I'll complete this instrumented research flow step by step. Starting with Step 1:", - "type": "text" - }, - { - "arguments": { - "query": "flue instrumentation" - }, - "id": "", - "name": "lookup", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "{\"id\":\"flue-session-2026\",\"query\":\"flue instrumentation\",\"topic\":\"session instrumentation\"}", - "type": "text" - } - ], - "isError": false, - "role": "toolResult", - "toolCallId": "", - "toolName": "lookup" - } - ], - "output": { - "content": [ - { - "thinking": "Good, I got the lookup result with id \"flue-session-2026\". Now I need to proceed to step 2: call web_search with this lookupId and the query \"Braintrust Flue reasoning stream instrumentation\".", - "thinkingSignature": "EpEDCm4IDhgCKkAMahve1kB1j4njp1ZSzO8eCK4bb5Ex7MB6tjdoOXfuUDRfkBqvIGdwVly8GiA8GGYC7Y/RVfREXF6A6OXHTwFyMhpjbGF1ZGUtc29ubmV0LTQtNS0yMDI1MDkyOTgAQgh0aGlua2luZxIMLw+fsa4yo7a7ZSoIGgyaTdvH020n2dQj9bUiMB4OokY29w+uZb4rqDnKDbLXUqUj6U0f1QCGQF2ZBYDDfDJyM5Jv1xY3rQiUUjkhBirQAS4XivjvMnWI5zHBmpI/u/pq4aGdrdb3vUsJ/ne6R1d80JA/INGF8vLkQcZ5xxEK0jU4yRE+ldc2uzavmtoxIKAFD6bqKks1Qv6u+TIZtuW7tUB/7jFxDw93KmJNruMIa8c1J6pPUyBy3xE7QwgmU0kDfIfdnljxLoEBiMwtnxjCPj9lT7HHz4FkrhbnQTV4MvAQohCLjaSbOTd6LdLxwtJPm9mc26ID4eYN1gCCzyAxGXhqLHlzc23PafvQjsZliWnLhaiNMnG0YEo78XkuJQsYAQ==", - "type": "thinking" - }, - { - "text": "Now proceeding to Step 2:", - "type": "text" - }, - { - "arguments": { - "lookupId": "flue-session-2026", - "query": "Braintrust Flue reasoning stream instrumentation" - }, - "id": "", - "name": "web_search", - "type": "toolCall" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "anthropic-messages", - "flue.input_message_offset": 1, - "flue.input_mode": "delta", - "flue.model": "claude-sonnet-4-5-20250929", - "flue.provider": "anthropic", - "flue.session": "main", - "flue.stop_reason": "toolUse", - "flue.turn_purpose": "agent", - "model": "claude-sonnet-4-5-20250929", - "provider": "anthropic" - }, - "metrics": { - "completion_tokens": 158, - "duration_ms": 0, - "estimated_cost": 0.0039606, - "prompt_cache_creation_tokens": 236, - "prompt_cached_tokens": 2222, - "prompt_tokens": 13, - "tokens": 2629 - } - }, - { - "name": "tool:web_search", - "type": "tool", - "children": [], - "input": { - "lookupId": "flue-session-2026", - "query": "Braintrust Flue reasoning stream instrumentation" - }, - "output": { - "content": [ - { - "text": "{\"lookupId\":\"flue-session-2026\",\"query\":\"Braintrust Flue reasoning stream instrumentation\",\"results\":[{\"title\":\"Flue reasoning stream instrumentation\",\"url\":\"https://example.test/flue/reasoning-streams\"}]}", - "type": "text" - } - ], - "details": { - "customTool": "web_search" - } - }, - "metadata": { - "flue.session": "main", - "flue.tool_name": "web_search", - "provider": "flue" - }, - "metrics": { - "duration_ms": 0 - } - }, - { - "name": "flue.turn", - "type": "llm", - "children": [], - "input": [ - { - "content": [ - { - "thinking": "Good, I got the lookup result with id \"flue-session-2026\". Now I need to proceed to step 2: call web_search with this lookupId and the query \"Braintrust Flue reasoning stream instrumentation\".", - "thinkingSignature": "EpEDCm4IDhgCKkAMahve1kB1j4njp1ZSzO8eCK4bb5Ex7MB6tjdoOXfuUDRfkBqvIGdwVly8GiA8GGYC7Y/RVfREXF6A6OXHTwFyMhpjbGF1ZGUtc29ubmV0LTQtNS0yMDI1MDkyOTgAQgh0aGlua2luZxIMLw+fsa4yo7a7ZSoIGgyaTdvH020n2dQj9bUiMB4OokY29w+uZb4rqDnKDbLXUqUj6U0f1QCGQF2ZBYDDfDJyM5Jv1xY3rQiUUjkhBirQAS4XivjvMnWI5zHBmpI/u/pq4aGdrdb3vUsJ/ne6R1d80JA/INGF8vLkQcZ5xxEK0jU4yRE+ldc2uzavmtoxIKAFD6bqKks1Qv6u+TIZtuW7tUB/7jFxDw93KmJNruMIa8c1J6pPUyBy3xE7QwgmU0kDfIfdnljxLoEBiMwtnxjCPj9lT7HHz4FkrhbnQTV4MvAQohCLjaSbOTd6LdLxwtJPm9mc26ID4eYN1gCCzyAxGXhqLHlzc23PafvQjsZliWnLhaiNMnG0YEo78XkuJQsYAQ==", - "type": "thinking" - }, - { - "text": "Now proceeding to Step 2:", - "type": "text" - }, - { - "arguments": { - "lookupId": "flue-session-2026", - "query": "Braintrust Flue reasoning stream instrumentation" - }, - "id": "", - "name": "web_search", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "{\"lookupId\":\"flue-session-2026\",\"query\":\"Braintrust Flue reasoning stream instrumentation\",\"results\":[{\"title\":\"Flue reasoning stream instrumentation\",\"url\":\"https://example.test/flue/reasoning-streams\"}]}", - "type": "text" - } - ], - "isError": false, - "role": "toolResult", - "toolCallId": "", - "toolName": "web_search" - } - ], - "output": { - "content": [ - { - "thinking": "Perfect! I got the web_search results with one result. The first (and only) result has the URL \"https://example.test/flue/reasoning-streams\". Now I need to proceed to step 3: call summarize_source with this URL.", - "thinkingSignature": "EqQDCm4IDhgCKkAXv7EDAhWMcrBz11bUd5qxlzyy03CAGQfRgLHDE0uKzLATh1o2sedd8GxLwwQqV0CmLRfNxsdK/ZDp3Lks9iysMhpjbGF1ZGUtc29ubmV0LTQtNS0yMDI1MDkyOTgAQgh0aGlua2luZxIMXDMniOy5Dy0KI8x7GgzgHs7BYJMs6/N0ur8iMPe0SwnHhP39OjEu7P8ASwK9ffwa3VP67QOlPzOstRRdEZNd0mq+BVjpmBXB+QcAzCrjAeAh/UO/d7SfFmXjqwA3//thoL/TTlGa6W+fdOCFRikn65JWT4ZdVFetfqD+9/QCP/nXEwMsLnTBAxC4EGryrgs2YT+OWwmEJ7rHKFeDs5djhwSb2J2ZJO1kRvD9s2vCCDkImygYBSIVgx1Y0CZJHhzn2Vf39v0Fbth2wDZa+OqxivV9NmDZEjmgcM+tTRx1CCnaww8LZEJQ68v/dUJy7hhD4MrNcQ62evJYPfT3+4EKKWJ/7HI7KXz91//+YAkph63XstbJMFJ3AldAFiq3ecauXt5r42N5gmCmwZX+sI48ENLGGAE=", - "type": "thinking" - }, - { - "text": "Now proceeding to Step 3:", - "type": "text" - }, - { - "arguments": { - "url": "https://example.test/flue/reasoning-streams" - }, - "id": "", - "name": "summarize_source", - "type": "toolCall" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "anthropic-messages", - "flue.input_message_offset": 3, - "flue.input_mode": "delta", - "flue.model": "claude-sonnet-4-5-20250929", - "flue.provider": "anthropic", - "flue.session": "main", - "flue.stop_reason": "toolUse", - "flue.turn_purpose": "agent", - "model": "claude-sonnet-4-5-20250929", - "provider": "anthropic" - }, - "metrics": { - "completion_tokens": 138, - "duration_ms": 0, - "estimated_cost": 0.0037239000000000005, - "prompt_cache_creation_tokens": 234, - "prompt_cached_tokens": 2458, - "prompt_tokens": 13, - "tokens": 2843 - } - }, - { - "name": "tool:summarize_source", - "type": "tool", - "children": [], - "input": { - "url": "https://example.test/flue/reasoning-streams" - }, - "output": { - "content": [ - { - "text": "{\"summary\":\"Flue emits reasoning, tool execution, and LLM turn events separately.\",\"url\":\"https://example.test/flue/reasoning-streams\"}", - "type": "text" - } - ], - "details": { - "customTool": "summarize_source" - } - }, - "metadata": { - "flue.session": "main", - "flue.tool_name": "summarize_source", - "provider": "flue" - }, - "metrics": { - "duration_ms": 0 - } - }, - { - "name": "flue.turn", - "type": "llm", - "children": [], - "input": [ - { - "content": [ - { - "thinking": "Perfect! I got the web_search results with one result. The first (and only) result has the URL \"https://example.test/flue/reasoning-streams\". Now I need to proceed to step 3: call summarize_source with this URL.", - "thinkingSignature": "EqQDCm4IDhgCKkAXv7EDAhWMcrBz11bUd5qxlzyy03CAGQfRgLHDE0uKzLATh1o2sedd8GxLwwQqV0CmLRfNxsdK/ZDp3Lks9iysMhpjbGF1ZGUtc29ubmV0LTQtNS0yMDI1MDkyOTgAQgh0aGlua2luZxIMXDMniOy5Dy0KI8x7GgzgHs7BYJMs6/N0ur8iMPe0SwnHhP39OjEu7P8ASwK9ffwa3VP67QOlPzOstRRdEZNd0mq+BVjpmBXB+QcAzCrjAeAh/UO/d7SfFmXjqwA3//thoL/TTlGa6W+fdOCFRikn65JWT4ZdVFetfqD+9/QCP/nXEwMsLnTBAxC4EGryrgs2YT+OWwmEJ7rHKFeDs5djhwSb2J2ZJO1kRvD9s2vCCDkImygYBSIVgx1Y0CZJHhzn2Vf39v0Fbth2wDZa+OqxivV9NmDZEjmgcM+tTRx1CCnaww8LZEJQ68v/dUJy7hhD4MrNcQ62evJYPfT3+4EKKWJ/7HI7KXz91//+YAkph63XstbJMFJ3AldAFiq3ecauXt5r42N5gmCmwZX+sI48ENLGGAE=", - "type": "thinking" - }, - { - "text": "Now proceeding to Step 3:", - "type": "text" - }, - { - "arguments": { - "url": "https://example.test/flue/reasoning-streams" - }, - "id": "", - "name": "summarize_source", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "{\"summary\":\"Flue emits reasoning, tool execution, and LLM turn events separately.\",\"url\":\"https://example.test/flue/reasoning-streams\"}", - "type": "text" - } - ], - "isError": false, - "role": "toolResult", - "toolCallId": "", - "toolName": "summarize_source" - } - ], - "output": { - "content": [ - { - "thinking": "Perfect! The summarize_source call has completed successfully. According to the instructions, I should now reply with exactly \"PROMPT_DONE\" and no other text.", - "thinkingSignature": "Eu8CCm4IDhgCKkAn2QOIIlYfvcs6ynsmhCslHDtBsrTA4CfmimK7zyz6+k6dZldV2JBtEBET41It0gFcGEKC0n0mAhaYVuLutpDDMhpjbGF1ZGUtc29ubmV0LTQtNS0yMDI1MDkyOTgAQgh0aGlua2luZxIM1xfLptxSF8rKzou8GgwRLuVmfiBYOT4CZIsiMAYz8TpvchMAWodP+T9uZ+dJ47MVdNv+p86nZXmT5d5EWlCroiAOFHHzJH2o0UjJQSquAX6wu+jIC8/DXwurGIIf2ToldPuJWF5HGj8kHMnwh777XS1NUFl1irM24rvP7kTAv3Dp8YEwb8t3g1EsicY7j+C5DzgwxVsPYCVOCgdMqQjtsqpL7AX5KQ9+hR1gK1DQjshv4Gze3hw/+C+9iptd3RlDIuwO0cm8NY+sVqeWVh3TtQujQlVt1rW/HfWjclrtT0LZSGwc6RypmfMMQi6lncIk/EPgy0XbS1plKW0tLBgB", - "type": "thinking" - }, - { - "text": "PROMPT_DONE", - "type": "text" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "anthropic-messages", - "flue.input_message_offset": 5, - "flue.input_mode": "delta", - "flue.model": "claude-sonnet-4-5-20250929", - "flue.provider": "anthropic", - "flue.session": "main", - "flue.stop_reason": "stop", - "flue.turn_purpose": "agent", - "model": "claude-sonnet-4-5-20250929", - "provider": "anthropic" - }, - "metrics": { - "completion_tokens": 51, - "duration_ms": 0, - "estimated_cost": 0.0023391, - "prompt_cache_creation_tokens": 194, - "prompt_cached_tokens": 2692, - "prompt_tokens": 13, - "tokens": 2950 - } - } - ], - "input": [ - { - "content": [ - { - "text": "Complete this instrumented research flow. Call exactly one tool per turn and wait for each tool result before choosing the next tool. Step 1: call lookup with query \"flue instrumentation\". Step 2: use the lookup result id as lookupId and call web_search with query \"Braintrust Flue reasoning stream instrumentation\". Step 3: use the first web_search result url and call summarize_source. After summarize_source returns, reply with exactly PROMPT_DONE and no other text.", - "type": "text" - } - ], - "role": "user" - } - ], - "output": "PROMPT_DONE", - "metadata": { - "flue.operation": "prompt", - "flue.session": "main", - "flue.workflow_name": "instrumentation", - "provider": "flue", - "scenario": "flue-instrumentation" - }, - "metrics": { - "duration_ms": 0 - } - } - ] -} diff --git a/e2e/scenarios/flue-instrumentation/__snapshots__/flue-v0-8-0-cli.span-tree.txt b/e2e/scenarios/flue-instrumentation/__snapshots__/flue-v0-8-0-cli.span-tree.txt deleted file mode 100644 index e90a4892a..000000000 --- a/e2e/scenarios/flue-instrumentation/__snapshots__/flue-v0-8-0-cli.span-tree.txt +++ /dev/null @@ -1,1335 +0,0 @@ -span_tree: -├── workflow:instrumentation [task] -│ input: { -│ "metadata": { -│ "scenario": "flue-instrumentation", -│ "testRunId": "" -│ }, -│ "scenario": "flue-instrumentation" -│ } -│ output: { -│ "scenario": "flue-instrumentation", -│ "status": "done" -│ } -│ metadata: { -│ "flue.workflow_name": "instrumentation", -│ "provider": "flue", -│ "scenario": "flue-instrumentation" -│ } -│ metrics: { -│ "duration_ms": 0 -│ } -│ ├── flue.skill [task] -│ │ input: [ -│ │ { -│ │ "content": [ -│ │ { -│ │ "text": "Run the skill named \"e2e-flue-skill\".\n\nArguments:\n{\n \"marker\": \"SKILL_DONE\"\n}", -│ │ "type": "text" -│ │ } -│ │ ], -│ │ "role": "user" -│ │ } -│ │ ] -│ │ output: "SKILL_DONE" -│ │ metadata: { -│ │ "flue.operation": "skill", -│ │ "flue.session": "skill", -│ │ "flue.workflow_name": "instrumentation", -│ │ "provider": "flue", -│ │ "scenario": "flue-instrumentation" -│ │ } -│ │ metrics: { -│ │ "duration_ms": 0 -│ │ } -│ │ ├── flue.turn [llm] -│ │ │ input: [ -│ │ │ { -│ │ │ "content": [ -│ │ │ { -│ │ │ "text": "Run the skill named \"e2e-flue-skill\".\n\nArguments:\n{\n \"marker\": \"SKILL_DONE\"\n}", -│ │ │ "type": "text" -│ │ │ } -│ │ │ ], -│ │ │ "role": "user" -│ │ │ } -│ │ │ ] -│ │ │ output: { -│ │ │ "content": [ -│ │ │ { -│ │ │ "text": "I'll run the e2e-flue-skill. Let me first read the skill file to understand its instructions.", -│ │ │ "type": "text" -│ │ │ }, -│ │ │ { -│ │ │ "arguments": { -│ │ │ "path": ".agents/e2e-flue-skill.md" -│ │ │ }, -│ │ │ "id": "", -│ │ │ "name": "read", -│ │ │ "type": "toolCall" -│ │ │ } -│ │ │ ], -│ │ │ "role": "assistant" -│ │ │ } -│ │ │ metadata: { -│ │ │ "flue.api": "anthropic-messages", -│ │ │ "flue.model": "claude-sonnet-4-5-20250929", -│ │ │ "flue.provider": "anthropic", -│ │ │ "flue.session": "skill", -│ │ │ "flue.stop_reason": "toolUse", -│ │ │ "flue.turn_purpose": "agent", -│ │ │ "model": "claude-sonnet-4-5-20250929", -│ │ │ "provider": "anthropic" -│ │ │ } -│ │ │ metrics: { -│ │ │ "completion_tokens": 89, -│ │ │ "duration_ms": 0, -│ │ │ "estimated_cost": 0.0041862, -│ │ │ "prompt_cache_creation_tokens": 670, -│ │ │ "prompt_cached_tokens": 1099, -│ │ │ "prompt_tokens": 3, -│ │ │ "tokens": 1861 -│ │ │ } -│ │ ├── tool:read [tool] -│ │ │ input: { -│ │ │ "path": ".agents/e2e-flue-skill.md" -│ │ │ } -│ │ │ output: { -│ │ │ "content": [ -│ │ │ { -│ │ │ "text": "ENOENT: no such file or directory, open '/e2e/scenarios/flue-instrumentation/.agents/e2e-flue-skill.md'", -│ │ │ "type": "text" -│ │ │ } -│ │ │ ], -│ │ │ "details": {} -│ │ │ } -│ │ │ metadata: { -│ │ │ "flue.session": "skill", -│ │ │ "flue.tool_name": "read", -│ │ │ "provider": "flue" -│ │ │ } -│ │ │ metrics: { -│ │ │ "duration_ms": 0 -│ │ │ } -│ │ │ error: "{\"content\":[{\"type\":\"text\",\"text\":\"ENOENT: no such file or directory, open '/e2e/scenarios/flue-instrumentation/.agents/e2e-flue-skill.md'\"}],\"details\":{}}" -│ │ ├── flue.turn [llm] -│ │ │ input: [ -│ │ │ { -│ │ │ "content": [ -│ │ │ { -│ │ │ "text": "Run the skill named \"e2e-flue-skill\".\n\nArguments:\n{\n \"marker\": \"SKILL_DONE\"\n}", -│ │ │ "type": "text" -│ │ │ } -│ │ │ ], -│ │ │ "role": "user" -│ │ │ }, -│ │ │ { -│ │ │ "content": [ -│ │ │ { -│ │ │ "text": "I'll run the e2e-flue-skill. Let me first read the skill file to understand its instructions.", -│ │ │ "type": "text" -│ │ │ }, -│ │ │ { -│ │ │ "arguments": { -│ │ │ "path": ".agents/e2e-flue-skill.md" -│ │ │ }, -│ │ │ "id": "", -│ │ │ "name": "read", -│ │ │ "type": "toolCall" -│ │ │ } -│ │ │ ], -│ │ │ "role": "assistant" -│ │ │ }, -│ │ │ { -│ │ │ "content": [ -│ │ │ { -│ │ │ "text": "ENOENT: no such file or directory, open '/e2e/scenarios/flue-instrumentation/.agents/e2e-flue-skill.md'", -│ │ │ "type": "text" -│ │ │ } -│ │ │ ], -│ │ │ "isError": true, -│ │ │ "role": "toolResult", -│ │ │ "toolCallId": "", -│ │ │ "toolName": "read" -│ │ │ } -│ │ │ ] -│ │ │ output: { -│ │ │ "content": [ -│ │ │ { -│ │ │ "arguments": { -│ │ │ "path": ".agents" -│ │ │ }, -│ │ │ "id": "", -│ │ │ "name": "read", -│ │ │ "type": "toolCall" -│ │ │ } -│ │ │ ], -│ │ │ "role": "assistant" -│ │ │ } -│ │ │ metadata: { -│ │ │ "flue.api": "anthropic-messages", -│ │ │ "flue.model": "claude-sonnet-4-5-20250929", -│ │ │ "flue.provider": "anthropic", -│ │ │ "flue.session": "skill", -│ │ │ "flue.stop_reason": "toolUse", -│ │ │ "flue.turn_purpose": "agent", -│ │ │ "model": "claude-sonnet-4-5-20250929", -│ │ │ "provider": "anthropic" -│ │ │ } -│ │ │ metrics: { -│ │ │ "completion_tokens": 52, -│ │ │ "duration_ms": 0, -│ │ │ "estimated_cost": 0.00201795, -│ │ │ "prompt_cache_creation_tokens": 183, -│ │ │ "prompt_cached_tokens": 1769, -│ │ │ "prompt_tokens": 7, -│ │ │ "tokens": 2011 -│ │ │ } -│ │ ├── tool:read [tool] -│ │ │ input: { -│ │ │ "path": ".agents" -│ │ │ } -│ │ │ output: { -│ │ │ "content": [ -│ │ │ { -│ │ │ "text": "skills", -│ │ │ "type": "text" -│ │ │ } -│ │ │ ], -│ │ │ "details": { -│ │ │ "entries": 1, -│ │ │ "isDirectory": true, -│ │ │ "path": ".agents" -│ │ │ } -│ │ │ } -│ │ │ metadata: { -│ │ │ "flue.session": "skill", -│ │ │ "flue.tool_name": "read", -│ │ │ "provider": "flue" -│ │ │ } -│ │ │ metrics: { -│ │ │ "duration_ms": 0 -│ │ │ } -│ │ ├── flue.turn [llm] -│ │ │ input: [ -│ │ │ { -│ │ │ "content": [ -│ │ │ { -│ │ │ "text": "Run the skill named \"e2e-flue-skill\".\n\nArguments:\n{\n \"marker\": \"SKILL_DONE\"\n}", -│ │ │ "type": "text" -│ │ │ } -│ │ │ ], -│ │ │ "role": "user" -│ │ │ }, -│ │ │ { -│ │ │ "content": [ -│ │ │ { -│ │ │ "text": "I'll run the e2e-flue-skill. Let me first read the skill file to understand its instructions.", -│ │ │ "type": "text" -│ │ │ }, -│ │ │ { -│ │ │ "arguments": { -│ │ │ "path": ".agents/e2e-flue-skill.md" -│ │ │ }, -│ │ │ "id": "", -│ │ │ "name": "read", -│ │ │ "type": "toolCall" -│ │ │ } -│ │ │ ], -│ │ │ "role": "assistant" -│ │ │ }, -│ │ │ { -│ │ │ "content": [ -│ │ │ { -│ │ │ "text": "ENOENT: no such file or directory, open '/e2e/scenarios/flue-instrumentation/.agents/e2e-flue-skill.md'", -│ │ │ "type": "text" -│ │ │ } -│ │ │ ], -│ │ │ "isError": true, -│ │ │ "role": "toolResult", -│ │ │ "toolCallId": "", -│ │ │ "toolName": "read" -│ │ │ }, -│ │ │ { -│ │ │ "content": [ -│ │ │ { -│ │ │ "arguments": { -│ │ │ "path": ".agents" -│ │ │ }, -│ │ │ "id": "", -│ │ │ "name": "read", -│ │ │ "type": "toolCall" -│ │ │ } -│ │ │ ], -│ │ │ "role": "assistant" -│ │ │ }, -│ │ │ { -│ │ │ "content": [ -│ │ │ { -│ │ │ "text": "skills", -│ │ │ "type": "text" -│ │ │ } -│ │ │ ], -│ │ │ "isError": false, -│ │ │ "role": "toolResult", -│ │ │ "toolCallId": "", -│ │ │ "toolName": "read" -│ │ │ } -│ │ │ ] -│ │ │ output: { -│ │ │ "content": [ -│ │ │ { -│ │ │ "arguments": { -│ │ │ "path": ".agents/skills" -│ │ │ }, -│ │ │ "id": "", -│ │ │ "name": "read", -│ │ │ "type": "toolCall" -│ │ │ } -│ │ │ ], -│ │ │ "role": "assistant" -│ │ │ } -│ │ │ metadata: { -│ │ │ "flue.api": "anthropic-messages", -│ │ │ "flue.model": "claude-sonnet-4-5-20250929", -│ │ │ "flue.provider": "anthropic", -│ │ │ "flue.session": "skill", -│ │ │ "flue.stop_reason": "toolUse", -│ │ │ "flue.turn_purpose": "agent", -│ │ │ "model": "claude-sonnet-4-5-20250929", -│ │ │ "provider": "anthropic" -│ │ │ } -│ │ │ metrics: { -│ │ │ "completion_tokens": 54, -│ │ │ "duration_ms": 0, -│ │ │ "estimated_cost": 0.0016611, -│ │ │ "prompt_cache_creation_tokens": 66, -│ │ │ "prompt_cached_tokens": 1952, -│ │ │ "prompt_tokens": 6, -│ │ │ "tokens": 2078 -│ │ │ } -│ │ ├── tool:read [tool] -│ │ │ input: { -│ │ │ "path": ".agents/skills" -│ │ │ } -│ │ │ output: { -│ │ │ "content": [ -│ │ │ { -│ │ │ "text": "e2e-flue-skill", -│ │ │ "type": "text" -│ │ │ } -│ │ │ ], -│ │ │ "details": { -│ │ │ "entries": 1, -│ │ │ "isDirectory": true, -│ │ │ "path": ".agents/skills" -│ │ │ } -│ │ │ } -│ │ │ metadata: { -│ │ │ "flue.session": "skill", -│ │ │ "flue.tool_name": "read", -│ │ │ "provider": "flue" -│ │ │ } -│ │ │ metrics: { -│ │ │ "duration_ms": 0 -│ │ │ } -│ │ ├── flue.turn [llm] -│ │ │ input: [ -│ │ │ { -│ │ │ "content": [ -│ │ │ { -│ │ │ "text": "Run the skill named \"e2e-flue-skill\".\n\nArguments:\n{\n \"marker\": \"SKILL_DONE\"\n}", -│ │ │ "type": "text" -│ │ │ } -│ │ │ ], -│ │ │ "role": "user" -│ │ │ }, -│ │ │ { -│ │ │ "content": [ -│ │ │ { -│ │ │ "text": "I'll run the e2e-flue-skill. Let me first read the skill file to understand its instructions.", -│ │ │ "type": "text" -│ │ │ }, -│ │ │ { -│ │ │ "arguments": { -│ │ │ "path": ".agents/e2e-flue-skill.md" -│ │ │ }, -│ │ │ "id": "", -│ │ │ "name": "read", -│ │ │ "type": "toolCall" -│ │ │ } -│ │ │ ], -│ │ │ "role": "assistant" -│ │ │ }, -│ │ │ { -│ │ │ "content": [ -│ │ │ { -│ │ │ "text": "ENOENT: no such file or directory, open '/e2e/scenarios/flue-instrumentation/.agents/e2e-flue-skill.md'", -│ │ │ "type": "text" -│ │ │ } -│ │ │ ], -│ │ │ "isError": true, -│ │ │ "role": "toolResult", -│ │ │ "toolCallId": "", -│ │ │ "toolName": "read" -│ │ │ }, -│ │ │ { -│ │ │ "content": [ -│ │ │ { -│ │ │ "arguments": { -│ │ │ "path": ".agents" -│ │ │ }, -│ │ │ "id": "", -│ │ │ "name": "read", -│ │ │ "type": "toolCall" -│ │ │ } -│ │ │ ], -│ │ │ "role": "assistant" -│ │ │ }, -│ │ │ { -│ │ │ "content": [ -│ │ │ { -│ │ │ "text": "skills", -│ │ │ "type": "text" -│ │ │ } -│ │ │ ], -│ │ │ "isError": false, -│ │ │ "role": "toolResult", -│ │ │ "toolCallId": "", -│ │ │ "toolName": "read" -│ │ │ }, -│ │ │ { -│ │ │ "content": [ -│ │ │ { -│ │ │ "arguments": { -│ │ │ "path": ".agents/skills" -│ │ │ }, -│ │ │ "id": "", -│ │ │ "name": "read", -│ │ │ "type": "toolCall" -│ │ │ } -│ │ │ ], -│ │ │ "role": "assistant" -│ │ │ }, -│ │ │ { -│ │ │ "content": [ -│ │ │ { -│ │ │ "text": "e2e-flue-skill", -│ │ │ "type": "text" -│ │ │ } -│ │ │ ], -│ │ │ "isError": false, -│ │ │ "role": "toolResult", -│ │ │ "toolCallId": "", -│ │ │ "toolName": "read" -│ │ │ } -│ │ │ ] -│ │ │ output: { -│ │ │ "content": [ -│ │ │ { -│ │ │ "arguments": { -│ │ │ "path": ".agents/skills/e2e-flue-skill" -│ │ │ }, -│ │ │ "id": "", -│ │ │ "name": "read", -│ │ │ "type": "toolCall" -│ │ │ } -│ │ │ ], -│ │ │ "role": "assistant" -│ │ │ } -│ │ │ metadata: { -│ │ │ "flue.api": "anthropic-messages", -│ │ │ "flue.model": "claude-sonnet-4-5-20250929", -│ │ │ "flue.provider": "anthropic", -│ │ │ "flue.session": "skill", -│ │ │ "flue.stop_reason": "toolUse", -│ │ │ "flue.turn_purpose": "agent", -│ │ │ "model": "claude-sonnet-4-5-20250929", -│ │ │ "provider": "anthropic" -│ │ │ } -│ │ │ metrics: { -│ │ │ "completion_tokens": 63, -│ │ │ "duration_ms": 0, -│ │ │ "estimated_cost": 0.0018459, -│ │ │ "prompt_cache_creation_tokens": 74, -│ │ │ "prompt_cached_tokens": 2018, -│ │ │ "prompt_tokens": 6, -│ │ │ "tokens": 2161 -│ │ │ } -│ │ ├── tool:read [tool] -│ │ │ input: { -│ │ │ "path": ".agents/skills/e2e-flue-skill" -│ │ │ } -│ │ │ output: { -│ │ │ "content": [ -│ │ │ { -│ │ │ "text": "SKILL.md", -│ │ │ "type": "text" -│ │ │ } -│ │ │ ], -│ │ │ "details": { -│ │ │ "entries": 1, -│ │ │ "isDirectory": true, -│ │ │ "path": ".agents/skills/e2e-flue-skill" -│ │ │ } -│ │ │ } -│ │ │ metadata: { -│ │ │ "flue.session": "skill", -│ │ │ "flue.tool_name": "read", -│ │ │ "provider": "flue" -│ │ │ } -│ │ │ metrics: { -│ │ │ "duration_ms": 0 -│ │ │ } -│ │ ├── flue.turn [llm] -│ │ │ input: [ -│ │ │ { -│ │ │ "content": [ -│ │ │ { -│ │ │ "text": "Run the skill named \"e2e-flue-skill\".\n\nArguments:\n{\n \"marker\": \"SKILL_DONE\"\n}", -│ │ │ "type": "text" -│ │ │ } -│ │ │ ], -│ │ │ "role": "user" -│ │ │ }, -│ │ │ { -│ │ │ "content": [ -│ │ │ { -│ │ │ "text": "I'll run the e2e-flue-skill. Let me first read the skill file to understand its instructions.", -│ │ │ "type": "text" -│ │ │ }, -│ │ │ { -│ │ │ "arguments": { -│ │ │ "path": ".agents/e2e-flue-skill.md" -│ │ │ }, -│ │ │ "id": "", -│ │ │ "name": "read", -│ │ │ "type": "toolCall" -│ │ │ } -│ │ │ ], -│ │ │ "role": "assistant" -│ │ │ }, -│ │ │ { -│ │ │ "content": [ -│ │ │ { -│ │ │ "text": "ENOENT: no such file or directory, open '/e2e/scenarios/flue-instrumentation/.agents/e2e-flue-skill.md'", -│ │ │ "type": "text" -│ │ │ } -│ │ │ ], -│ │ │ "isError": true, -│ │ │ "role": "toolResult", -│ │ │ "toolCallId": "", -│ │ │ "toolName": "read" -│ │ │ }, -│ │ │ { -│ │ │ "content": [ -│ │ │ { -│ │ │ "arguments": { -│ │ │ "path": ".agents" -│ │ │ }, -│ │ │ "id": "", -│ │ │ "name": "read", -│ │ │ "type": "toolCall" -│ │ │ } -│ │ │ ], -│ │ │ "role": "assistant" -│ │ │ }, -│ │ │ { -│ │ │ "content": [ -│ │ │ { -│ │ │ "text": "skills", -│ │ │ "type": "text" -│ │ │ } -│ │ │ ], -│ │ │ "isError": false, -│ │ │ "role": "toolResult", -│ │ │ "toolCallId": "", -│ │ │ "toolName": "read" -│ │ │ }, -│ │ │ { -│ │ │ "content": [ -│ │ │ { -│ │ │ "arguments": { -│ │ │ "path": ".agents/skills" -│ │ │ }, -│ │ │ "id": "", -│ │ │ "name": "read", -│ │ │ "type": "toolCall" -│ │ │ } -│ │ │ ], -│ │ │ "role": "assistant" -│ │ │ }, -│ │ │ { -│ │ │ "content": [ -│ │ │ { -│ │ │ "text": "e2e-flue-skill", -│ │ │ "type": "text" -│ │ │ } -│ │ │ ], -│ │ │ "isError": false, -│ │ │ "role": "toolResult", -│ │ │ "toolCallId": "", -│ │ │ "toolName": "read" -│ │ │ }, -│ │ │ { -│ │ │ "content": [ -│ │ │ { -│ │ │ "arguments": { -│ │ │ "path": ".agents/skills/e2e-flue-skill" -│ │ │ }, -│ │ │ "id": "", -│ │ │ "name": "read", -│ │ │ "type": "toolCall" -│ │ │ } -│ │ │ ], -│ │ │ "role": "assistant" -│ │ │ }, -│ │ │ { -│ │ │ "content": [ -│ │ │ { -│ │ │ "text": "SKILL.md", -│ │ │ "type": "text" -│ │ │ } -│ │ │ ], -│ │ │ "isError": false, -│ │ │ "role": "toolResult", -│ │ │ "toolCallId": "", -│ │ │ "toolName": "read" -│ │ │ } -│ │ │ ] -│ │ │ output: { -│ │ │ "content": [ -│ │ │ { -│ │ │ "arguments": { -│ │ │ "path": ".agents/skills/e2e-flue-skill/SKILL.md" -│ │ │ }, -│ │ │ "id": "", -│ │ │ "name": "read", -│ │ │ "type": "toolCall" -│ │ │ } -│ │ │ ], -│ │ │ "role": "assistant" -│ │ │ } -│ │ │ metadata: { -│ │ │ "flue.api": "anthropic-messages", -│ │ │ "flue.model": "claude-sonnet-4-5-20250929", -│ │ │ "flue.provider": "anthropic", -│ │ │ "flue.session": "skill", -│ │ │ "flue.stop_reason": "toolUse", -│ │ │ "flue.turn_purpose": "agent", -│ │ │ "model": "claude-sonnet-4-5-20250929", -│ │ │ "provider": "anthropic" -│ │ │ } -│ │ │ metrics: { -│ │ │ "completion_tokens": 68, -│ │ │ "duration_ms": 0, -│ │ │ "estimated_cost": 0.0019618500000000002, -│ │ │ "prompt_cache_creation_tokens": 79, -│ │ │ "prompt_cached_tokens": 2092, -│ │ │ "prompt_tokens": 6, -│ │ │ "tokens": 2245 -│ │ │ } -│ │ ├── tool:read [tool] -│ │ │ input: { -│ │ │ "path": ".agents/skills/e2e-flue-skill/SKILL.md" -│ │ │ } -│ │ │ output: { -│ │ │ "content": [ -│ │ │ { -│ │ │ "text": "---\nname: e2e-flue-skill\ndescription: Deterministic skill used by the Braintrust Flue instrumentation e2e test.\n---\n\nReturn the marker from the provided args exactly once. Output no other text.\n", -│ │ │ "type": "text" -│ │ │ } -│ │ │ ], -│ │ │ "details": { -│ │ │ "lines": 7, -│ │ │ "path": ".agents/skills/e2e-flue-skill/SKILL.md" -│ │ │ } -│ │ │ } -│ │ │ metadata: { -│ │ │ "flue.session": "skill", -│ │ │ "flue.tool_name": "read", -│ │ │ "provider": "flue" -│ │ │ } -│ │ │ metrics: { -│ │ │ "duration_ms": 0 -│ │ │ } -│ │ └── flue.turn [llm] -│ │ input: [ -│ │ { -│ │ "content": [ -│ │ { -│ │ "text": "Run the skill named \"e2e-flue-skill\".\n\nArguments:\n{\n \"marker\": \"SKILL_DONE\"\n}", -│ │ "type": "text" -│ │ } -│ │ ], -│ │ "role": "user" -│ │ }, -│ │ { -│ │ "content": [ -│ │ { -│ │ "text": "I'll run the e2e-flue-skill. Let me first read the skill file to understand its instructions.", -│ │ "type": "text" -│ │ }, -│ │ { -│ │ "arguments": { -│ │ "path": ".agents/e2e-flue-skill.md" -│ │ }, -│ │ "id": "", -│ │ "name": "read", -│ │ "type": "toolCall" -│ │ } -│ │ ], -│ │ "role": "assistant" -│ │ }, -│ │ { -│ │ "content": [ -│ │ { -│ │ "text": "ENOENT: no such file or directory, open '/e2e/scenarios/flue-instrumentation/.agents/e2e-flue-skill.md'", -│ │ "type": "text" -│ │ } -│ │ ], -│ │ "isError": true, -│ │ "role": "toolResult", -│ │ "toolCallId": "", -│ │ "toolName": "read" -│ │ }, -│ │ { -│ │ "content": [ -│ │ { -│ │ "arguments": { -│ │ "path": ".agents" -│ │ }, -│ │ "id": "", -│ │ "name": "read", -│ │ "type": "toolCall" -│ │ } -│ │ ], -│ │ "role": "assistant" -│ │ }, -│ │ { -│ │ "content": [ -│ │ { -│ │ "text": "skills", -│ │ "type": "text" -│ │ } -│ │ ], -│ │ "isError": false, -│ │ "role": "toolResult", -│ │ "toolCallId": "", -│ │ "toolName": "read" -│ │ }, -│ │ { -│ │ "content": [ -│ │ { -│ │ "arguments": { -│ │ "path": ".agents/skills" -│ │ }, -│ │ "id": "", -│ │ "name": "read", -│ │ "type": "toolCall" -│ │ } -│ │ ], -│ │ "role": "assistant" -│ │ }, -│ │ { -│ │ "content": [ -│ │ { -│ │ "text": "e2e-flue-skill", -│ │ "type": "text" -│ │ } -│ │ ], -│ │ "isError": false, -│ │ "role": "toolResult", -│ │ "toolCallId": "", -│ │ "toolName": "read" -│ │ }, -│ │ { -│ │ "content": [ -│ │ { -│ │ "arguments": { -│ │ "path": ".agents/skills/e2e-flue-skill" -│ │ }, -│ │ "id": "", -│ │ "name": "read", -│ │ "type": "toolCall" -│ │ } -│ │ ], -│ │ "role": "assistant" -│ │ }, -│ │ { -│ │ "content": [ -│ │ { -│ │ "text": "SKILL.md", -│ │ "type": "text" -│ │ } -│ │ ], -│ │ "isError": false, -│ │ "role": "toolResult", -│ │ "toolCallId": "", -│ │ "toolName": "read" -│ │ }, -│ │ { -│ │ "content": [ -│ │ { -│ │ "arguments": { -│ │ "path": ".agents/skills/e2e-flue-skill/SKILL.md" -│ │ }, -│ │ "id": "", -│ │ "name": "read", -│ │ "type": "toolCall" -│ │ } -│ │ ], -│ │ "role": "assistant" -│ │ }, -│ │ { -│ │ "content": [ -│ │ { -│ │ "text": "---\nname: e2e-flue-skill\ndescription: Deterministic skill used by the Braintrust Flue instrumentation e2e test.\n---\n\nReturn the marker from the provided args exactly once. Output no other text.\n", -│ │ "type": "text" -│ │ } -│ │ ], -│ │ "isError": false, -│ │ "role": "toolResult", -│ │ "toolCallId": "", -│ │ "toolName": "read" -│ │ } -│ │ ] -│ │ output: { -│ │ "content": [ -│ │ { -│ │ "text": "SKILL_DONE", -│ │ "type": "text" -│ │ } -│ │ ], -│ │ "role": "assistant" -│ │ } -│ │ metadata: { -│ │ "flue.api": "anthropic-messages", -│ │ "flue.model": "claude-sonnet-4-5-20250929", -│ │ "flue.provider": "anthropic", -│ │ "flue.session": "skill", -│ │ "flue.stop_reason": "stop", -│ │ "flue.turn_purpose": "agent", -│ │ "model": "claude-sonnet-4-5-20250929", -│ │ "provider": "anthropic" -│ │ } -│ │ metrics: { -│ │ "completion_tokens": 8, -│ │ "duration_ms": 0, -│ │ "estimated_cost": 0.00129555, -│ │ "prompt_cache_creation_tokens": 135, -│ │ "prompt_cached_tokens": 2171, -│ │ "prompt_tokens": 6, -│ │ "tokens": 2320 -│ │ } -│ ├── flue.task [task] -│ │ input: "Reply with exactly TASK_DONE and no other text." -│ │ output: "TASK_DONE" -│ │ metadata: { -│ │ "flue.session": "task", -│ │ "provider": "flue" -│ │ } -│ │ metrics: { -│ │ "duration_ms": 0 -│ │ } -│ │ └── flue.turn [llm] -│ │ input: [ -│ │ { -│ │ "content": [ -│ │ { -│ │ "text": "Reply with exactly TASK_DONE and no other text.", -│ │ "type": "text" -│ │ } -│ │ ], -│ │ "role": "user" -│ │ } -│ │ ] -│ │ output: { -│ │ "content": [ -│ │ { -│ │ "text": "TASK_DONE", -│ │ "textSignature": "{\"v\":1,\"id\":\"msg_09b32b2821e9a4d8016a184600019081929d34bb7d4c6a246f\"}", -│ │ "type": "text" -│ │ } -│ │ ], -│ │ "role": "assistant" -│ │ } -│ │ metadata: { -│ │ "flue.api": "openai-responses", -│ │ "flue.input_mode": "full", -│ │ "flue.model": "gpt-4o-mini", -│ │ "flue.provider": "openai", -│ │ "flue.session": "task:task:", -│ │ "flue.stop_reason": "stop", -│ │ "flue.turn_purpose": "agent", -│ │ "model": "gpt-4o-mini", -│ │ "provider": "openai" -│ │ } -│ │ metrics: { -│ │ "completion_tokens": 4, -│ │ "duration_ms": 0, -│ │ "estimated_cost": 0.00012299999999999998, -│ │ "prompt_cache_creation_tokens": 0, -│ │ "prompt_cached_tokens": 0, -│ │ "prompt_tokens": 804, -│ │ "tokens": 808 -│ │ } -│ └── flue.compact [task] -│ input: { -│ "estimatedTokens": 2950, -│ "reason": "manual" -│ } -│ output: { -│ "completed": true -│ } -│ metadata: { -│ "flue.operation": "compact", -│ "flue.session": "main", -│ "flue.workflow_name": "instrumentation", -│ "provider": "flue", -│ "scenario": "flue-instrumentation" -│ } -│ metrics: { -│ "duration_ms": 0 -│ } -│ └── compaction:manual [task] -│ input: { -│ "estimatedTokens": 2950, -│ "reason": "manual" -│ } -│ output: { -│ "messagesAfter": 2, -│ "messagesBefore": 8 -│ } -│ metadata: { -│ "flue.compaction_reason": "manual", -│ "flue.session": "main", -│ "provider": "flue" -│ } -│ metrics: { -│ "duration_ms": 0, -│ "messages_after": 2, -│ "messages_before": 8 -│ } -│ └── flue.turn [llm] -│ input: [ -│ { -│ "content": [ -│ { -│ "text": "\n[User]: Complete this instrumented research flow. Call exactly one tool per turn and wait for each tool result before choosing the next tool. Step 1: call lookup with query \"flue instrumentation\". Step 2: use the lookup result id as lookupId and call web_search with query \"Braintrust Flue reasoning stream instrumentation\". Step 3: use the first web_search result url and call summarize_source. After summarize_source returns, reply with exactly PROMPT_DONE and no other text.\n\n[Assistant thinking]: The user wants me to complete a research flow with exactly three steps:\n1. Call lookup with query \"flue instrumentation\"\n2. Call web_search with the lookupId from step 1 and query \"Braintrust Flue reasoning stream instrumentation\"\n3. Call summarize_source with the first URL from step 2\n4. Reply with exactly \"PROMPT_DONE\"\n\nThe user emphasizes I should call exactly one tool per turn and wait for each result. Let me start with step 1.\n\n[Assistant]: I'll complete this instrumented research flow step by step. Starting with Step 1:\n\n[Assistant tool calls]: lookup(query=\"flue instrumentation\")\n\n[Tool result]: {\"id\":\"flue-session-2026\",\"query\":\"flue instrumentation\",\"topic\":\"session instrumentation\"}\n\n[Assistant thinking]: Good, I got the lookup result with id \"flue-session-2026\". Now I need to proceed to step 2: call web_search with this lookupId and the query \"Braintrust Flue reasoning stream instrumentation\".\n\n[Assistant]: Now proceeding to Step 2:\n\n[Assistant tool calls]: web_search(lookupId=\"flue-session-2026\", query=\"Braintrust Flue reasoning stream instrumentation\")\n\n[Tool result]: {\"lookupId\":\"flue-session-2026\",\"query\":\"Braintrust Flue reasoning stream instrumentation\",\"results\":[{\"title\":\"Flue reasoning stream instrumentation\",\"url\":\"https://example.test/flue/reasoning-streams\"}]}\n\n[Assistant thinking]: Perfect! I got the web_search results with one result. The first (and only) result has the URL \"https://example.test/flue/reasoning-streams\". Now I need to proceed to step 3: call summarize_source with this URL.\n\n[Assistant]: Now proceeding to Step 3:\n\n[Assistant tool calls]: summarize_source(url=\"https://example.test/flue/reasoning-streams\")\n\n[Tool result]: {\"summary\":\"Flue emits reasoning, tool execution, and LLM turn events separately.\",\"url\":\"https://example.test/flue/reasoning-streams\"}\n\n\nThis is the PREFIX of a turn that was too large to keep. The SUFFIX (recent work) is retained.\n\nSummarize the prefix to provide context for the retained suffix:\n\n## Original Request\n[What did the user ask for in this turn?]\n\n## Early Progress\n- [Key decisions and work done in the prefix]\n\n## Context for Suffix\n- [Information needed to understand the retained recent work]\n\nBe concise. Focus on what's needed to understand the kept suffix.", -│ "type": "text" -│ } -│ ], -│ "role": "user" -│ } -│ ] -│ output: { -│ "content": [ -│ { -│ "text": "## Original Request\nThe user requested to complete an instrumented research flow consisting of three specific steps, involving calling various tools in a sequential manner.\n\n## Early Progress", -│ "textSignature": "{\"v\":1,\"id\":\"msg_01ecf68325f9e202016a184601309081a0878ef4054e950d32\"}", -│ "type": "text" -│ } -│ ], -│ "role": "assistant" -│ } -│ metadata: { -│ "flue.api": "openai-responses", -│ "flue.model": "gpt-4o-mini", -│ "flue.provider": "openai", -│ "flue.session": "main", -│ "flue.stop_reason": "stop", -│ "flue.turn_purpose": "compaction_prefix", -│ "model": "gpt-4o-mini", -│ "provider": "openai" -│ } -│ metrics: { -│ "completion_tokens": 0, -│ "duration_ms": 0, -│ "estimated_cost": 0, -│ "prompt_cache_creation_tokens": 0, -│ "prompt_cached_tokens": 0, -│ "prompt_tokens": 0, -│ "tokens": 0 -│ } -└── flue.prompt [task] - input: [ - { - "content": [ - { - "text": "Complete this instrumented research flow. Call exactly one tool per turn and wait for each tool result before choosing the next tool. Step 1: call lookup with query \"flue instrumentation\". Step 2: use the lookup result id as lookupId and call web_search with query \"Braintrust Flue reasoning stream instrumentation\". Step 3: use the first web_search result url and call summarize_source. After summarize_source returns, reply with exactly PROMPT_DONE and no other text.", - "type": "text" - } - ], - "role": "user" - } - ] - output: "PROMPT_DONE" - metadata: { - "flue.operation": "prompt", - "flue.session": "main", - "flue.workflow_name": "instrumentation", - "provider": "flue", - "scenario": "flue-instrumentation" - } - metrics: { - "duration_ms": 0 - } - ├── flue.turn [llm] - │ input: [ - │ { - │ "content": [ - │ { - │ "text": "Complete this instrumented research flow. Call exactly one tool per turn and wait for each tool result before choosing the next tool. Step 1: call lookup with query \"flue instrumentation\". Step 2: use the lookup result id as lookupId and call web_search with query \"Braintrust Flue reasoning stream instrumentation\". Step 3: use the first web_search result url and call summarize_source. After summarize_source returns, reply with exactly PROMPT_DONE and no other text.", - │ "type": "text" - │ } - │ ], - │ "role": "user" - │ } - │ ] - │ output: { - │ "content": [ - │ { - │ "thinking": "The user wants me to complete a research flow with exactly three steps:\n1. Call lookup with query \"flue instrumentation\"\n2. Call web_search with the lookupId from step 1 and query \"Braintrust Flue reasoning stream instrumentation\"\n3. Call summarize_source with the first URL from step 2\n4. Reply with exactly \"PROMPT_DONE\"\n\nThe user emphasizes I should call exactly one tool per turn and wait for each result. Let me start with step 1.", - │ "thinkingSignature": "EoQFCm4IDhgCKkD+AoKMhpXNPUJ5eANwM28uUCJPCE+AXh0ACw7aXniX+YN04zdJl3hlHMbtAuKhhczLiEeOJtu4KtVpea5u5NFdMhpjbGF1ZGUtc29ubmV0LTQtNS0yMDI1MDkyOTgAQgh0aGlua2luZxIMIOTWCNEtsiI9vF4SGgwPXC861uZx8lpOziEiMLsCukGIluD0U44degzYnHFc+BwXNIEB+yatSHlZE+a5smr9crZcft3OLmsIdXuaFSrDAyQxWd9oqfJdnvza9eWIaOCeNkVCjrTc/wotBwh5Xp8m9k/WKt7P4zzUJBuLQ1f6ZnsdWkL8OiL/2E7Up55z2+UuVWsg1Y90/x27YRFdQjv3RnztYp4bZ5oIHAlTJoCzWiBYEh/Z8c2O+wy6MHo1cp0T57OroB/4HAqRZxrjTOXpKYLmYKXAY5tU8DTd3A0/FfEhg3oPSbOhX+112szFCpm3BRNJYooUUhwSu1hO4H/6r7d/Yp9XIM4tBfzuDx83CC/YetdOFgnYkdliw0xaUGaj4o37rWIRctWHAMdlUc4NDA0Ljc1uirHDXkN8EJwWSoIjPSBTA/p/wgLrvRV5Yy9Z0dMoWOpgYF6t1H/tNAyY8ml1CA7IWyd3jnfTZ1/G3Y0xFDwJDFlb4/QKc67w36SSCa+X9mMoTxD/DYN249lsII+XX3lQ7F1zJ07THOtCyRirCaRhmE/kGwl5tiB1QHrWa9bNHxQxf9i92435JGKN053Fe14AR70MtBSzIU2/zbgvhB1vF5lZt2i46hNxfo4lXR495gEwdwC6+8HJnlEJAJkdeHfZKj+RiwXoiYKPKsSbHi83X8Hh32IifuglNxdlxVQYAQ==", - │ "type": "thinking" - │ }, - │ { - │ "text": "I'll complete this instrumented research flow step by step. Starting with Step 1:", - │ "type": "text" - │ }, - │ { - │ "arguments": { - │ "query": "flue instrumentation" - │ }, - │ "id": "", - │ "name": "lookup", - │ "type": "toolCall" - │ } - │ ], - │ "role": "assistant" - │ } - │ metadata: { - │ "flue.api": "anthropic-messages", - │ "flue.input_mode": "full", - │ "flue.model": "claude-sonnet-4-5-20250929", - │ "flue.provider": "anthropic", - │ "flue.session": "main", - │ "flue.stop_reason": "toolUse", - │ "flue.turn_purpose": "agent", - │ "model": "claude-sonnet-4-5-20250929", - │ "provider": "anthropic" - │ } - │ metrics: { - │ "completion_tokens": 196, - │ "duration_ms": 0, - │ "estimated_cost": 0.0113025, - │ "prompt_cache_creation_tokens": 2222, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 10, - │ "tokens": 2428 - │ } - ├── tool:lookup [tool] - │ input: { - │ "query": "flue instrumentation" - │ } - │ output: { - │ "content": [ - │ { - │ "text": "{\"id\":\"flue-session-2026\",\"query\":\"flue instrumentation\",\"topic\":\"session instrumentation\"}", - │ "type": "text" - │ } - │ ], - │ "details": { - │ "customTool": "lookup" - │ } - │ } - │ metadata: { - │ "flue.session": "main", - │ "flue.tool_name": "lookup", - │ "provider": "flue" - │ } - │ metrics: { - │ "duration_ms": 0 - │ } - ├── flue.turn [llm] - │ input: [ - │ { - │ "content": [ - │ { - │ "thinking": "The user wants me to complete a research flow with exactly three steps:\n1. Call lookup with query \"flue instrumentation\"\n2. Call web_search with the lookupId from step 1 and query \"Braintrust Flue reasoning stream instrumentation\"\n3. Call summarize_source with the first URL from step 2\n4. Reply with exactly \"PROMPT_DONE\"\n\nThe user emphasizes I should call exactly one tool per turn and wait for each result. Let me start with step 1.", - │ "thinkingSignature": "EoQFCm4IDhgCKkD+AoKMhpXNPUJ5eANwM28uUCJPCE+AXh0ACw7aXniX+YN04zdJl3hlHMbtAuKhhczLiEeOJtu4KtVpea5u5NFdMhpjbGF1ZGUtc29ubmV0LTQtNS0yMDI1MDkyOTgAQgh0aGlua2luZxIMIOTWCNEtsiI9vF4SGgwPXC861uZx8lpOziEiMLsCukGIluD0U44degzYnHFc+BwXNIEB+yatSHlZE+a5smr9crZcft3OLmsIdXuaFSrDAyQxWd9oqfJdnvza9eWIaOCeNkVCjrTc/wotBwh5Xp8m9k/WKt7P4zzUJBuLQ1f6ZnsdWkL8OiL/2E7Up55z2+UuVWsg1Y90/x27YRFdQjv3RnztYp4bZ5oIHAlTJoCzWiBYEh/Z8c2O+wy6MHo1cp0T57OroB/4HAqRZxrjTOXpKYLmYKXAY5tU8DTd3A0/FfEhg3oPSbOhX+112szFCpm3BRNJYooUUhwSu1hO4H/6r7d/Yp9XIM4tBfzuDx83CC/YetdOFgnYkdliw0xaUGaj4o37rWIRctWHAMdlUc4NDA0Ljc1uirHDXkN8EJwWSoIjPSBTA/p/wgLrvRV5Yy9Z0dMoWOpgYF6t1H/tNAyY8ml1CA7IWyd3jnfTZ1/G3Y0xFDwJDFlb4/QKc67w36SSCa+X9mMoTxD/DYN249lsII+XX3lQ7F1zJ07THOtCyRirCaRhmE/kGwl5tiB1QHrWa9bNHxQxf9i92435JGKN053Fe14AR70MtBSzIU2/zbgvhB1vF5lZt2i46hNxfo4lXR495gEwdwC6+8HJnlEJAJkdeHfZKj+RiwXoiYKPKsSbHi83X8Hh32IifuglNxdlxVQYAQ==", - │ "type": "thinking" - │ }, - │ { - │ "text": "I'll complete this instrumented research flow step by step. Starting with Step 1:", - │ "type": "text" - │ }, - │ { - │ "arguments": { - │ "query": "flue instrumentation" - │ }, - │ "id": "", - │ "name": "lookup", - │ "type": "toolCall" - │ } - │ ], - │ "role": "assistant" - │ }, - │ { - │ "content": [ - │ { - │ "text": "{\"id\":\"flue-session-2026\",\"query\":\"flue instrumentation\",\"topic\":\"session instrumentation\"}", - │ "type": "text" - │ } - │ ], - │ "isError": false, - │ "role": "toolResult", - │ "toolCallId": "", - │ "toolName": "lookup" - │ } - │ ] - │ output: { - │ "content": [ - │ { - │ "thinking": "Good, I got the lookup result with id \"flue-session-2026\". Now I need to proceed to step 2: call web_search with this lookupId and the query \"Braintrust Flue reasoning stream instrumentation\".", - │ "thinkingSignature": "EpEDCm4IDhgCKkAMahve1kB1j4njp1ZSzO8eCK4bb5Ex7MB6tjdoOXfuUDRfkBqvIGdwVly8GiA8GGYC7Y/RVfREXF6A6OXHTwFyMhpjbGF1ZGUtc29ubmV0LTQtNS0yMDI1MDkyOTgAQgh0aGlua2luZxIMLw+fsa4yo7a7ZSoIGgyaTdvH020n2dQj9bUiMB4OokY29w+uZb4rqDnKDbLXUqUj6U0f1QCGQF2ZBYDDfDJyM5Jv1xY3rQiUUjkhBirQAS4XivjvMnWI5zHBmpI/u/pq4aGdrdb3vUsJ/ne6R1d80JA/INGF8vLkQcZ5xxEK0jU4yRE+ldc2uzavmtoxIKAFD6bqKks1Qv6u+TIZtuW7tUB/7jFxDw93KmJNruMIa8c1J6pPUyBy3xE7QwgmU0kDfIfdnljxLoEBiMwtnxjCPj9lT7HHz4FkrhbnQTV4MvAQohCLjaSbOTd6LdLxwtJPm9mc26ID4eYN1gCCzyAxGXhqLHlzc23PafvQjsZliWnLhaiNMnG0YEo78XkuJQsYAQ==", - │ "type": "thinking" - │ }, - │ { - │ "text": "Now proceeding to Step 2:", - │ "type": "text" - │ }, - │ { - │ "arguments": { - │ "lookupId": "flue-session-2026", - │ "query": "Braintrust Flue reasoning stream instrumentation" - │ }, - │ "id": "", - │ "name": "web_search", - │ "type": "toolCall" - │ } - │ ], - │ "role": "assistant" - │ } - │ metadata: { - │ "flue.api": "anthropic-messages", - │ "flue.input_message_offset": 1, - │ "flue.input_mode": "delta", - │ "flue.model": "claude-sonnet-4-5-20250929", - │ "flue.provider": "anthropic", - │ "flue.session": "main", - │ "flue.stop_reason": "toolUse", - │ "flue.turn_purpose": "agent", - │ "model": "claude-sonnet-4-5-20250929", - │ "provider": "anthropic" - │ } - │ metrics: { - │ "completion_tokens": 158, - │ "duration_ms": 0, - │ "estimated_cost": 0.0039606, - │ "prompt_cache_creation_tokens": 236, - │ "prompt_cached_tokens": 2222, - │ "prompt_tokens": 13, - │ "tokens": 2629 - │ } - ├── tool:web_search [tool] - │ input: { - │ "lookupId": "flue-session-2026", - │ "query": "Braintrust Flue reasoning stream instrumentation" - │ } - │ output: { - │ "content": [ - │ { - │ "text": "{\"lookupId\":\"flue-session-2026\",\"query\":\"Braintrust Flue reasoning stream instrumentation\",\"results\":[{\"title\":\"Flue reasoning stream instrumentation\",\"url\":\"https://example.test/flue/reasoning-streams\"}]}", - │ "type": "text" - │ } - │ ], - │ "details": { - │ "customTool": "web_search" - │ } - │ } - │ metadata: { - │ "flue.session": "main", - │ "flue.tool_name": "web_search", - │ "provider": "flue" - │ } - │ metrics: { - │ "duration_ms": 0 - │ } - ├── flue.turn [llm] - │ input: [ - │ { - │ "content": [ - │ { - │ "thinking": "Good, I got the lookup result with id \"flue-session-2026\". Now I need to proceed to step 2: call web_search with this lookupId and the query \"Braintrust Flue reasoning stream instrumentation\".", - │ "thinkingSignature": "EpEDCm4IDhgCKkAMahve1kB1j4njp1ZSzO8eCK4bb5Ex7MB6tjdoOXfuUDRfkBqvIGdwVly8GiA8GGYC7Y/RVfREXF6A6OXHTwFyMhpjbGF1ZGUtc29ubmV0LTQtNS0yMDI1MDkyOTgAQgh0aGlua2luZxIMLw+fsa4yo7a7ZSoIGgyaTdvH020n2dQj9bUiMB4OokY29w+uZb4rqDnKDbLXUqUj6U0f1QCGQF2ZBYDDfDJyM5Jv1xY3rQiUUjkhBirQAS4XivjvMnWI5zHBmpI/u/pq4aGdrdb3vUsJ/ne6R1d80JA/INGF8vLkQcZ5xxEK0jU4yRE+ldc2uzavmtoxIKAFD6bqKks1Qv6u+TIZtuW7tUB/7jFxDw93KmJNruMIa8c1J6pPUyBy3xE7QwgmU0kDfIfdnljxLoEBiMwtnxjCPj9lT7HHz4FkrhbnQTV4MvAQohCLjaSbOTd6LdLxwtJPm9mc26ID4eYN1gCCzyAxGXhqLHlzc23PafvQjsZliWnLhaiNMnG0YEo78XkuJQsYAQ==", - │ "type": "thinking" - │ }, - │ { - │ "text": "Now proceeding to Step 2:", - │ "type": "text" - │ }, - │ { - │ "arguments": { - │ "lookupId": "flue-session-2026", - │ "query": "Braintrust Flue reasoning stream instrumentation" - │ }, - │ "id": "", - │ "name": "web_search", - │ "type": "toolCall" - │ } - │ ], - │ "role": "assistant" - │ }, - │ { - │ "content": [ - │ { - │ "text": "{\"lookupId\":\"flue-session-2026\",\"query\":\"Braintrust Flue reasoning stream instrumentation\",\"results\":[{\"title\":\"Flue reasoning stream instrumentation\",\"url\":\"https://example.test/flue/reasoning-streams\"}]}", - │ "type": "text" - │ } - │ ], - │ "isError": false, - │ "role": "toolResult", - │ "toolCallId": "", - │ "toolName": "web_search" - │ } - │ ] - │ output: { - │ "content": [ - │ { - │ "thinking": "Perfect! I got the web_search results with one result. The first (and only) result has the URL \"https://example.test/flue/reasoning-streams\". Now I need to proceed to step 3: call summarize_source with this URL.", - │ "thinkingSignature": "EqQDCm4IDhgCKkAXv7EDAhWMcrBz11bUd5qxlzyy03CAGQfRgLHDE0uKzLATh1o2sedd8GxLwwQqV0CmLRfNxsdK/ZDp3Lks9iysMhpjbGF1ZGUtc29ubmV0LTQtNS0yMDI1MDkyOTgAQgh0aGlua2luZxIMXDMniOy5Dy0KI8x7GgzgHs7BYJMs6/N0ur8iMPe0SwnHhP39OjEu7P8ASwK9ffwa3VP67QOlPzOstRRdEZNd0mq+BVjpmBXB+QcAzCrjAeAh/UO/d7SfFmXjqwA3//thoL/TTlGa6W+fdOCFRikn65JWT4ZdVFetfqD+9/QCP/nXEwMsLnTBAxC4EGryrgs2YT+OWwmEJ7rHKFeDs5djhwSb2J2ZJO1kRvD9s2vCCDkImygYBSIVgx1Y0CZJHhzn2Vf39v0Fbth2wDZa+OqxivV9NmDZEjmgcM+tTRx1CCnaww8LZEJQ68v/dUJy7hhD4MrNcQ62evJYPfT3+4EKKWJ/7HI7KXz91//+YAkph63XstbJMFJ3AldAFiq3ecauXt5r42N5gmCmwZX+sI48ENLGGAE=", - │ "type": "thinking" - │ }, - │ { - │ "text": "Now proceeding to Step 3:", - │ "type": "text" - │ }, - │ { - │ "arguments": { - │ "url": "https://example.test/flue/reasoning-streams" - │ }, - │ "id": "", - │ "name": "summarize_source", - │ "type": "toolCall" - │ } - │ ], - │ "role": "assistant" - │ } - │ metadata: { - │ "flue.api": "anthropic-messages", - │ "flue.input_message_offset": 3, - │ "flue.input_mode": "delta", - │ "flue.model": "claude-sonnet-4-5-20250929", - │ "flue.provider": "anthropic", - │ "flue.session": "main", - │ "flue.stop_reason": "toolUse", - │ "flue.turn_purpose": "agent", - │ "model": "claude-sonnet-4-5-20250929", - │ "provider": "anthropic" - │ } - │ metrics: { - │ "completion_tokens": 138, - │ "duration_ms": 0, - │ "estimated_cost": 0.0037239000000000005, - │ "prompt_cache_creation_tokens": 234, - │ "prompt_cached_tokens": 2458, - │ "prompt_tokens": 13, - │ "tokens": 2843 - │ } - ├── tool:summarize_source [tool] - │ input: { - │ "url": "https://example.test/flue/reasoning-streams" - │ } - │ output: { - │ "content": [ - │ { - │ "text": "{\"summary\":\"Flue emits reasoning, tool execution, and LLM turn events separately.\",\"url\":\"https://example.test/flue/reasoning-streams\"}", - │ "type": "text" - │ } - │ ], - │ "details": { - │ "customTool": "summarize_source" - │ } - │ } - │ metadata: { - │ "flue.session": "main", - │ "flue.tool_name": "summarize_source", - │ "provider": "flue" - │ } - │ metrics: { - │ "duration_ms": 0 - │ } - └── flue.turn [llm] - input: [ - { - "content": [ - { - "thinking": "Perfect! I got the web_search results with one result. The first (and only) result has the URL \"https://example.test/flue/reasoning-streams\". Now I need to proceed to step 3: call summarize_source with this URL.", - "thinkingSignature": "EqQDCm4IDhgCKkAXv7EDAhWMcrBz11bUd5qxlzyy03CAGQfRgLHDE0uKzLATh1o2sedd8GxLwwQqV0CmLRfNxsdK/ZDp3Lks9iysMhpjbGF1ZGUtc29ubmV0LTQtNS0yMDI1MDkyOTgAQgh0aGlua2luZxIMXDMniOy5Dy0KI8x7GgzgHs7BYJMs6/N0ur8iMPe0SwnHhP39OjEu7P8ASwK9ffwa3VP67QOlPzOstRRdEZNd0mq+BVjpmBXB+QcAzCrjAeAh/UO/d7SfFmXjqwA3//thoL/TTlGa6W+fdOCFRikn65JWT4ZdVFetfqD+9/QCP/nXEwMsLnTBAxC4EGryrgs2YT+OWwmEJ7rHKFeDs5djhwSb2J2ZJO1kRvD9s2vCCDkImygYBSIVgx1Y0CZJHhzn2Vf39v0Fbth2wDZa+OqxivV9NmDZEjmgcM+tTRx1CCnaww8LZEJQ68v/dUJy7hhD4MrNcQ62evJYPfT3+4EKKWJ/7HI7KXz91//+YAkph63XstbJMFJ3AldAFiq3ecauXt5r42N5gmCmwZX+sI48ENLGGAE=", - "type": "thinking" - }, - { - "text": "Now proceeding to Step 3:", - "type": "text" - }, - { - "arguments": { - "url": "https://example.test/flue/reasoning-streams" - }, - "id": "", - "name": "summarize_source", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "{\"summary\":\"Flue emits reasoning, tool execution, and LLM turn events separately.\",\"url\":\"https://example.test/flue/reasoning-streams\"}", - "type": "text" - } - ], - "isError": false, - "role": "toolResult", - "toolCallId": "", - "toolName": "summarize_source" - } - ] - output: { - "content": [ - { - "thinking": "Perfect! The summarize_source call has completed successfully. According to the instructions, I should now reply with exactly \"PROMPT_DONE\" and no other text.", - "thinkingSignature": "Eu8CCm4IDhgCKkAn2QOIIlYfvcs6ynsmhCslHDtBsrTA4CfmimK7zyz6+k6dZldV2JBtEBET41It0gFcGEKC0n0mAhaYVuLutpDDMhpjbGF1ZGUtc29ubmV0LTQtNS0yMDI1MDkyOTgAQgh0aGlua2luZxIM1xfLptxSF8rKzou8GgwRLuVmfiBYOT4CZIsiMAYz8TpvchMAWodP+T9uZ+dJ47MVdNv+p86nZXmT5d5EWlCroiAOFHHzJH2o0UjJQSquAX6wu+jIC8/DXwurGIIf2ToldPuJWF5HGj8kHMnwh777XS1NUFl1irM24rvP7kTAv3Dp8YEwb8t3g1EsicY7j+C5DzgwxVsPYCVOCgdMqQjtsqpL7AX5KQ9+hR1gK1DQjshv4Gze3hw/+C+9iptd3RlDIuwO0cm8NY+sVqeWVh3TtQujQlVt1rW/HfWjclrtT0LZSGwc6RypmfMMQi6lncIk/EPgy0XbS1plKW0tLBgB", - "type": "thinking" - }, - { - "text": "PROMPT_DONE", - "type": "text" - } - ], - "role": "assistant" - } - metadata: { - "flue.api": "anthropic-messages", - "flue.input_message_offset": 5, - "flue.input_mode": "delta", - "flue.model": "claude-sonnet-4-5-20250929", - "flue.provider": "anthropic", - "flue.session": "main", - "flue.stop_reason": "stop", - "flue.turn_purpose": "agent", - "model": "claude-sonnet-4-5-20250929", - "provider": "anthropic" - } - metrics: { - "completion_tokens": 51, - "duration_ms": 0, - "estimated_cost": 0.0023391, - "prompt_cache_creation_tokens": 194, - "prompt_cached_tokens": 2692, - "prompt_tokens": 13, - "tokens": 2950 - } diff --git a/e2e/scenarios/flue-instrumentation/__snapshots__/flue-v0-8-0-explicit.span-tree.json b/e2e/scenarios/flue-instrumentation/__snapshots__/flue-v0-8-0-explicit.span-tree.json deleted file mode 100644 index dd92abe29..000000000 --- a/e2e/scenarios/flue-instrumentation/__snapshots__/flue-v0-8-0-explicit.span-tree.json +++ /dev/null @@ -1,1448 +0,0 @@ -{ - "span_tree": [ - { - "name": "workflow:instrumentation", - "type": "task", - "children": [ - { - "name": "flue.skill", - "type": "task", - "children": [ - { - "name": "flue.turn", - "type": "llm", - "children": [], - "input": [ - { - "content": [ - { - "text": "Run the skill named \"e2e-flue-skill\".\n\nArguments:\n{\n \"marker\": \"SKILL_DONE\"\n}", - "type": "text" - } - ], - "role": "user" - } - ], - "output": { - "content": [ - { - "text": "I'll run the e2e-flue-skill. Let me first read the skill file to understand its instructions.", - "type": "text" - }, - { - "arguments": { - "path": ".agents/e2e-flue-skill.md" - }, - "id": "", - "name": "read", - "type": "toolCall" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "anthropic-messages", - "flue.model": "claude-sonnet-4-5-20250929", - "flue.provider": "anthropic", - "flue.session": "skill", - "flue.stop_reason": "toolUse", - "flue.turn_purpose": "agent", - "model": "claude-sonnet-4-5-20250929", - "provider": "anthropic" - }, - "metrics": { - "completion_tokens": 89, - "duration_ms": 0, - "estimated_cost": 0.0041862, - "prompt_cache_creation_tokens": 670, - "prompt_cached_tokens": 1099, - "prompt_tokens": 3, - "tokens": 1861 - } - }, - { - "name": "tool:read", - "type": "tool", - "children": [], - "input": { - "path": ".agents/e2e-flue-skill.md" - }, - "output": { - "content": [ - { - "text": "ENOENT: no such file or directory, open '/e2e/scenarios/flue-instrumentation/.agents/e2e-flue-skill.md'", - "type": "text" - } - ], - "details": {} - }, - "metadata": { - "flue.session": "skill", - "flue.tool_name": "read", - "provider": "flue" - }, - "metrics": { - "duration_ms": 0 - }, - "error": "{\"content\":[{\"type\":\"text\",\"text\":\"ENOENT: no such file or directory, open '/e2e/scenarios/flue-instrumentation/.agents/e2e-flue-skill.md'\"}],\"details\":{}}" - }, - { - "name": "flue.turn", - "type": "llm", - "children": [], - "input": [ - { - "content": [ - { - "text": "Run the skill named \"e2e-flue-skill\".\n\nArguments:\n{\n \"marker\": \"SKILL_DONE\"\n}", - "type": "text" - } - ], - "role": "user" - }, - { - "content": [ - { - "text": "I'll run the e2e-flue-skill. Let me first read the skill file to understand its instructions.", - "type": "text" - }, - { - "arguments": { - "path": ".agents/e2e-flue-skill.md" - }, - "id": "", - "name": "read", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "ENOENT: no such file or directory, open '/e2e/scenarios/flue-instrumentation/.agents/e2e-flue-skill.md'", - "type": "text" - } - ], - "isError": true, - "role": "toolResult", - "toolCallId": "", - "toolName": "read" - } - ], - "output": { - "content": [ - { - "arguments": { - "path": ".agents" - }, - "id": "", - "name": "read", - "type": "toolCall" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "anthropic-messages", - "flue.model": "claude-sonnet-4-5-20250929", - "flue.provider": "anthropic", - "flue.session": "skill", - "flue.stop_reason": "toolUse", - "flue.turn_purpose": "agent", - "model": "claude-sonnet-4-5-20250929", - "provider": "anthropic" - }, - "metrics": { - "completion_tokens": 52, - "duration_ms": 0, - "estimated_cost": 0.00201795, - "prompt_cache_creation_tokens": 183, - "prompt_cached_tokens": 1769, - "prompt_tokens": 7, - "tokens": 2011 - } - }, - { - "name": "tool:read", - "type": "tool", - "children": [], - "input": { - "path": ".agents" - }, - "output": { - "content": [ - { - "text": "skills", - "type": "text" - } - ], - "details": { - "entries": 1, - "isDirectory": true, - "path": ".agents" - } - }, - "metadata": { - "flue.session": "skill", - "flue.tool_name": "read", - "provider": "flue" - }, - "metrics": { - "duration_ms": 0 - } - }, - { - "name": "flue.turn", - "type": "llm", - "children": [], - "input": [ - { - "content": [ - { - "text": "Run the skill named \"e2e-flue-skill\".\n\nArguments:\n{\n \"marker\": \"SKILL_DONE\"\n}", - "type": "text" - } - ], - "role": "user" - }, - { - "content": [ - { - "text": "I'll run the e2e-flue-skill. Let me first read the skill file to understand its instructions.", - "type": "text" - }, - { - "arguments": { - "path": ".agents/e2e-flue-skill.md" - }, - "id": "", - "name": "read", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "ENOENT: no such file or directory, open '/e2e/scenarios/flue-instrumentation/.agents/e2e-flue-skill.md'", - "type": "text" - } - ], - "isError": true, - "role": "toolResult", - "toolCallId": "", - "toolName": "read" - }, - { - "content": [ - { - "arguments": { - "path": ".agents" - }, - "id": "", - "name": "read", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "skills", - "type": "text" - } - ], - "isError": false, - "role": "toolResult", - "toolCallId": "", - "toolName": "read" - } - ], - "output": { - "content": [ - { - "arguments": { - "path": ".agents/skills" - }, - "id": "", - "name": "read", - "type": "toolCall" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "anthropic-messages", - "flue.model": "claude-sonnet-4-5-20250929", - "flue.provider": "anthropic", - "flue.session": "skill", - "flue.stop_reason": "toolUse", - "flue.turn_purpose": "agent", - "model": "claude-sonnet-4-5-20250929", - "provider": "anthropic" - }, - "metrics": { - "completion_tokens": 54, - "duration_ms": 0, - "estimated_cost": 0.0016611, - "prompt_cache_creation_tokens": 66, - "prompt_cached_tokens": 1952, - "prompt_tokens": 6, - "tokens": 2078 - } - }, - { - "name": "tool:read", - "type": "tool", - "children": [], - "input": { - "path": ".agents/skills" - }, - "output": { - "content": [ - { - "text": "e2e-flue-skill", - "type": "text" - } - ], - "details": { - "entries": 1, - "isDirectory": true, - "path": ".agents/skills" - } - }, - "metadata": { - "flue.session": "skill", - "flue.tool_name": "read", - "provider": "flue" - }, - "metrics": { - "duration_ms": 0 - } - }, - { - "name": "flue.turn", - "type": "llm", - "children": [], - "input": [ - { - "content": [ - { - "text": "Run the skill named \"e2e-flue-skill\".\n\nArguments:\n{\n \"marker\": \"SKILL_DONE\"\n}", - "type": "text" - } - ], - "role": "user" - }, - { - "content": [ - { - "text": "I'll run the e2e-flue-skill. Let me first read the skill file to understand its instructions.", - "type": "text" - }, - { - "arguments": { - "path": ".agents/e2e-flue-skill.md" - }, - "id": "", - "name": "read", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "ENOENT: no such file or directory, open '/e2e/scenarios/flue-instrumentation/.agents/e2e-flue-skill.md'", - "type": "text" - } - ], - "isError": true, - "role": "toolResult", - "toolCallId": "", - "toolName": "read" - }, - { - "content": [ - { - "arguments": { - "path": ".agents" - }, - "id": "", - "name": "read", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "skills", - "type": "text" - } - ], - "isError": false, - "role": "toolResult", - "toolCallId": "", - "toolName": "read" - }, - { - "content": [ - { - "arguments": { - "path": ".agents/skills" - }, - "id": "", - "name": "read", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "e2e-flue-skill", - "type": "text" - } - ], - "isError": false, - "role": "toolResult", - "toolCallId": "", - "toolName": "read" - } - ], - "output": { - "content": [ - { - "arguments": { - "path": ".agents/skills/e2e-flue-skill" - }, - "id": "", - "name": "read", - "type": "toolCall" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "anthropic-messages", - "flue.model": "claude-sonnet-4-5-20250929", - "flue.provider": "anthropic", - "flue.session": "skill", - "flue.stop_reason": "toolUse", - "flue.turn_purpose": "agent", - "model": "claude-sonnet-4-5-20250929", - "provider": "anthropic" - }, - "metrics": { - "completion_tokens": 63, - "duration_ms": 0, - "estimated_cost": 0.0018459, - "prompt_cache_creation_tokens": 74, - "prompt_cached_tokens": 2018, - "prompt_tokens": 6, - "tokens": 2161 - } - }, - { - "name": "tool:read", - "type": "tool", - "children": [], - "input": { - "path": ".agents/skills/e2e-flue-skill" - }, - "output": { - "content": [ - { - "text": "SKILL.md", - "type": "text" - } - ], - "details": { - "entries": 1, - "isDirectory": true, - "path": ".agents/skills/e2e-flue-skill" - } - }, - "metadata": { - "flue.session": "skill", - "flue.tool_name": "read", - "provider": "flue" - }, - "metrics": { - "duration_ms": 0 - } - }, - { - "name": "flue.turn", - "type": "llm", - "children": [], - "input": [ - { - "content": [ - { - "text": "Run the skill named \"e2e-flue-skill\".\n\nArguments:\n{\n \"marker\": \"SKILL_DONE\"\n}", - "type": "text" - } - ], - "role": "user" - }, - { - "content": [ - { - "text": "I'll run the e2e-flue-skill. Let me first read the skill file to understand its instructions.", - "type": "text" - }, - { - "arguments": { - "path": ".agents/e2e-flue-skill.md" - }, - "id": "", - "name": "read", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "ENOENT: no such file or directory, open '/e2e/scenarios/flue-instrumentation/.agents/e2e-flue-skill.md'", - "type": "text" - } - ], - "isError": true, - "role": "toolResult", - "toolCallId": "", - "toolName": "read" - }, - { - "content": [ - { - "arguments": { - "path": ".agents" - }, - "id": "", - "name": "read", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "skills", - "type": "text" - } - ], - "isError": false, - "role": "toolResult", - "toolCallId": "", - "toolName": "read" - }, - { - "content": [ - { - "arguments": { - "path": ".agents/skills" - }, - "id": "", - "name": "read", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "e2e-flue-skill", - "type": "text" - } - ], - "isError": false, - "role": "toolResult", - "toolCallId": "", - "toolName": "read" - }, - { - "content": [ - { - "arguments": { - "path": ".agents/skills/e2e-flue-skill" - }, - "id": "", - "name": "read", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "SKILL.md", - "type": "text" - } - ], - "isError": false, - "role": "toolResult", - "toolCallId": "", - "toolName": "read" - } - ], - "output": { - "content": [ - { - "arguments": { - "path": ".agents/skills/e2e-flue-skill/SKILL.md" - }, - "id": "", - "name": "read", - "type": "toolCall" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "anthropic-messages", - "flue.model": "claude-sonnet-4-5-20250929", - "flue.provider": "anthropic", - "flue.session": "skill", - "flue.stop_reason": "toolUse", - "flue.turn_purpose": "agent", - "model": "claude-sonnet-4-5-20250929", - "provider": "anthropic" - }, - "metrics": { - "completion_tokens": 68, - "duration_ms": 0, - "estimated_cost": 0.0019618500000000002, - "prompt_cache_creation_tokens": 79, - "prompt_cached_tokens": 2092, - "prompt_tokens": 6, - "tokens": 2245 - } - }, - { - "name": "tool:read", - "type": "tool", - "children": [], - "input": { - "path": ".agents/skills/e2e-flue-skill/SKILL.md" - }, - "output": { - "content": [ - { - "text": "---\nname: e2e-flue-skill\ndescription: Deterministic skill used by the Braintrust Flue instrumentation e2e test.\n---\n\nReturn the marker from the provided args exactly once. Output no other text.\n", - "type": "text" - } - ], - "details": { - "lines": 7, - "path": ".agents/skills/e2e-flue-skill/SKILL.md" - } - }, - "metadata": { - "flue.session": "skill", - "flue.tool_name": "read", - "provider": "flue" - }, - "metrics": { - "duration_ms": 0 - } - }, - { - "name": "flue.turn", - "type": "llm", - "children": [], - "input": [ - { - "content": [ - { - "text": "Run the skill named \"e2e-flue-skill\".\n\nArguments:\n{\n \"marker\": \"SKILL_DONE\"\n}", - "type": "text" - } - ], - "role": "user" - }, - { - "content": [ - { - "text": "I'll run the e2e-flue-skill. Let me first read the skill file to understand its instructions.", - "type": "text" - }, - { - "arguments": { - "path": ".agents/e2e-flue-skill.md" - }, - "id": "", - "name": "read", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "ENOENT: no such file or directory, open '/e2e/scenarios/flue-instrumentation/.agents/e2e-flue-skill.md'", - "type": "text" - } - ], - "isError": true, - "role": "toolResult", - "toolCallId": "", - "toolName": "read" - }, - { - "content": [ - { - "arguments": { - "path": ".agents" - }, - "id": "", - "name": "read", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "skills", - "type": "text" - } - ], - "isError": false, - "role": "toolResult", - "toolCallId": "", - "toolName": "read" - }, - { - "content": [ - { - "arguments": { - "path": ".agents/skills" - }, - "id": "", - "name": "read", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "e2e-flue-skill", - "type": "text" - } - ], - "isError": false, - "role": "toolResult", - "toolCallId": "", - "toolName": "read" - }, - { - "content": [ - { - "arguments": { - "path": ".agents/skills/e2e-flue-skill" - }, - "id": "", - "name": "read", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "SKILL.md", - "type": "text" - } - ], - "isError": false, - "role": "toolResult", - "toolCallId": "", - "toolName": "read" - }, - { - "content": [ - { - "arguments": { - "path": ".agents/skills/e2e-flue-skill/SKILL.md" - }, - "id": "", - "name": "read", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "---\nname: e2e-flue-skill\ndescription: Deterministic skill used by the Braintrust Flue instrumentation e2e test.\n---\n\nReturn the marker from the provided args exactly once. Output no other text.\n", - "type": "text" - } - ], - "isError": false, - "role": "toolResult", - "toolCallId": "", - "toolName": "read" - } - ], - "output": { - "content": [ - { - "text": "SKILL_DONE", - "type": "text" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "anthropic-messages", - "flue.model": "claude-sonnet-4-5-20250929", - "flue.provider": "anthropic", - "flue.session": "skill", - "flue.stop_reason": "stop", - "flue.turn_purpose": "agent", - "model": "claude-sonnet-4-5-20250929", - "provider": "anthropic" - }, - "metrics": { - "completion_tokens": 8, - "duration_ms": 0, - "estimated_cost": 0.00129555, - "prompt_cache_creation_tokens": 135, - "prompt_cached_tokens": 2171, - "prompt_tokens": 6, - "tokens": 2320 - } - } - ], - "input": [ - { - "content": [ - { - "text": "Run the skill named \"e2e-flue-skill\".\n\nArguments:\n{\n \"marker\": \"SKILL_DONE\"\n}", - "type": "text" - } - ], - "role": "user" - } - ], - "output": "SKILL_DONE", - "metadata": { - "flue.operation": "skill", - "flue.session": "skill", - "flue.workflow_name": "instrumentation", - "provider": "flue", - "scenario": "flue-instrumentation" - }, - "metrics": { - "duration_ms": 0 - } - }, - { - "name": "flue.task", - "type": "task", - "children": [ - { - "name": "flue.turn", - "type": "llm", - "children": [], - "input": [ - { - "content": [ - { - "text": "Reply with exactly TASK_DONE and no other text.", - "type": "text" - } - ], - "role": "user" - } - ], - "output": { - "content": [ - { - "text": "TASK_DONE", - "textSignature": "{\"v\":1,\"id\":\"msg_09b32b2821e9a4d8016a184600019081929d34bb7d4c6a246f\"}", - "type": "text" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "openai-responses", - "flue.input_mode": "full", - "flue.model": "gpt-4o-mini", - "flue.provider": "openai", - "flue.session": "task:task:", - "flue.stop_reason": "stop", - "flue.turn_purpose": "agent", - "model": "gpt-4o-mini", - "provider": "openai" - }, - "metrics": { - "completion_tokens": 4, - "duration_ms": 0, - "estimated_cost": 0.00012299999999999998, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 804, - "tokens": 808 - } - } - ], - "input": "Reply with exactly TASK_DONE and no other text.", - "output": "TASK_DONE", - "metadata": { - "flue.session": "task", - "provider": "flue" - }, - "metrics": { - "duration_ms": 0 - } - }, - { - "name": "flue.compact", - "type": "task", - "children": [ - { - "name": "compaction:manual", - "type": "task", - "children": [ - { - "name": "flue.turn", - "type": "llm", - "children": [], - "input": [ - { - "content": [ - { - "text": "\n[User]: Complete this instrumented research flow. Call exactly one tool per turn and wait for each tool result before choosing the next tool. Step 1: call lookup with query \"flue instrumentation\". Step 2: use the lookup result id as lookupId and call web_search with query \"Braintrust Flue reasoning stream instrumentation\". Step 3: use the first web_search result url and call summarize_source. After summarize_source returns, reply with exactly PROMPT_DONE and no other text.\n\n[Assistant thinking]: The user wants me to complete a research flow with exactly three steps:\n1. Call lookup with query \"flue instrumentation\"\n2. Call web_search with the lookupId from step 1 and query \"Braintrust Flue reasoning stream instrumentation\"\n3. Call summarize_source with the first URL from step 2\n4. Reply with exactly \"PROMPT_DONE\"\n\nThe user emphasizes I should call exactly one tool per turn and wait for each result. Let me start with step 1.\n\n[Assistant]: I'll complete this instrumented research flow step by step. Starting with Step 1:\n\n[Assistant tool calls]: lookup(query=\"flue instrumentation\")\n\n[Tool result]: {\"id\":\"flue-session-2026\",\"query\":\"flue instrumentation\",\"topic\":\"session instrumentation\"}\n\n[Assistant thinking]: Good, I got the lookup result with id \"flue-session-2026\". Now I need to proceed to step 2: call web_search with this lookupId and the query \"Braintrust Flue reasoning stream instrumentation\".\n\n[Assistant]: Now proceeding to Step 2:\n\n[Assistant tool calls]: web_search(lookupId=\"flue-session-2026\", query=\"Braintrust Flue reasoning stream instrumentation\")\n\n[Tool result]: {\"lookupId\":\"flue-session-2026\",\"query\":\"Braintrust Flue reasoning stream instrumentation\",\"results\":[{\"title\":\"Flue reasoning stream instrumentation\",\"url\":\"https://example.test/flue/reasoning-streams\"}]}\n\n[Assistant thinking]: Perfect! I got the web_search results with one result. The first (and only) result has the URL \"https://example.test/flue/reasoning-streams\". Now I need to proceed to step 3: call summarize_source with this URL.\n\n[Assistant]: Now proceeding to Step 3:\n\n[Assistant tool calls]: summarize_source(url=\"https://example.test/flue/reasoning-streams\")\n\n[Tool result]: {\"summary\":\"Flue emits reasoning, tool execution, and LLM turn events separately.\",\"url\":\"https://example.test/flue/reasoning-streams\"}\n\n\nThis is the PREFIX of a turn that was too large to keep. The SUFFIX (recent work) is retained.\n\nSummarize the prefix to provide context for the retained suffix:\n\n## Original Request\n[What did the user ask for in this turn?]\n\n## Early Progress\n- [Key decisions and work done in the prefix]\n\n## Context for Suffix\n- [Information needed to understand the retained recent work]\n\nBe concise. Focus on what's needed to understand the kept suffix.", - "type": "text" - } - ], - "role": "user" - } - ], - "output": { - "content": [ - { - "text": "## Original Request\nThe user requested to complete an instrumented research flow consisting of three specific steps, involving calling various tools in a sequential manner.\n\n## Early Progress", - "textSignature": "{\"v\":1,\"id\":\"msg_01ecf68325f9e202016a184601309081a0878ef4054e950d32\"}", - "type": "text" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "openai-responses", - "flue.model": "gpt-4o-mini", - "flue.provider": "openai", - "flue.session": "main", - "flue.stop_reason": "stop", - "flue.turn_purpose": "compaction_prefix", - "model": "gpt-4o-mini", - "provider": "openai" - }, - "metrics": { - "completion_tokens": 0, - "duration_ms": 0, - "estimated_cost": 0, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 0, - "tokens": 0 - } - } - ], - "input": { - "estimatedTokens": 2950, - "reason": "manual" - }, - "output": { - "messagesAfter": 2, - "messagesBefore": 8 - }, - "metadata": { - "flue.compaction_reason": "manual", - "flue.session": "main", - "provider": "flue" - }, - "metrics": { - "duration_ms": 0, - "messages_after": 2, - "messages_before": 8 - } - } - ], - "input": { - "estimatedTokens": 2950, - "reason": "manual" - }, - "output": { - "completed": true - }, - "metadata": { - "flue.operation": "compact", - "flue.session": "main", - "flue.workflow_name": "instrumentation", - "provider": "flue", - "scenario": "flue-instrumentation" - }, - "metrics": { - "duration_ms": 0 - } - } - ], - "input": { - "metadata": { - "scenario": "flue-instrumentation", - "testRunId": "" - }, - "scenario": "flue-instrumentation" - }, - "output": { - "scenario": "flue-instrumentation", - "status": "done" - }, - "metadata": { - "flue.workflow_name": "instrumentation", - "provider": "flue", - "scenario": "flue-instrumentation" - }, - "metrics": { - "duration_ms": 0 - } - }, - { - "name": "flue.prompt", - "type": "task", - "children": [ - { - "name": "flue.turn", - "type": "llm", - "children": [], - "input": [ - { - "content": [ - { - "text": "Complete this instrumented research flow. Call exactly one tool per turn and wait for each tool result before choosing the next tool. Step 1: call lookup with query \"flue instrumentation\". Step 2: use the lookup result id as lookupId and call web_search with query \"Braintrust Flue reasoning stream instrumentation\". Step 3: use the first web_search result url and call summarize_source. After summarize_source returns, reply with exactly PROMPT_DONE and no other text.", - "type": "text" - } - ], - "role": "user" - } - ], - "output": { - "content": [ - { - "thinking": "The user wants me to complete a research flow with exactly three steps:\n1. Call lookup with query \"flue instrumentation\"\n2. Call web_search with the lookupId from step 1 and query \"Braintrust Flue reasoning stream instrumentation\"\n3. Call summarize_source with the first URL from step 2\n4. Reply with exactly \"PROMPT_DONE\"\n\nThe user emphasizes I should call exactly one tool per turn and wait for each result. Let me start with step 1.", - "thinkingSignature": "EoQFCm4IDhgCKkD+AoKMhpXNPUJ5eANwM28uUCJPCE+AXh0ACw7aXniX+YN04zdJl3hlHMbtAuKhhczLiEeOJtu4KtVpea5u5NFdMhpjbGF1ZGUtc29ubmV0LTQtNS0yMDI1MDkyOTgAQgh0aGlua2luZxIMIOTWCNEtsiI9vF4SGgwPXC861uZx8lpOziEiMLsCukGIluD0U44degzYnHFc+BwXNIEB+yatSHlZE+a5smr9crZcft3OLmsIdXuaFSrDAyQxWd9oqfJdnvza9eWIaOCeNkVCjrTc/wotBwh5Xp8m9k/WKt7P4zzUJBuLQ1f6ZnsdWkL8OiL/2E7Up55z2+UuVWsg1Y90/x27YRFdQjv3RnztYp4bZ5oIHAlTJoCzWiBYEh/Z8c2O+wy6MHo1cp0T57OroB/4HAqRZxrjTOXpKYLmYKXAY5tU8DTd3A0/FfEhg3oPSbOhX+112szFCpm3BRNJYooUUhwSu1hO4H/6r7d/Yp9XIM4tBfzuDx83CC/YetdOFgnYkdliw0xaUGaj4o37rWIRctWHAMdlUc4NDA0Ljc1uirHDXkN8EJwWSoIjPSBTA/p/wgLrvRV5Yy9Z0dMoWOpgYF6t1H/tNAyY8ml1CA7IWyd3jnfTZ1/G3Y0xFDwJDFlb4/QKc67w36SSCa+X9mMoTxD/DYN249lsII+XX3lQ7F1zJ07THOtCyRirCaRhmE/kGwl5tiB1QHrWa9bNHxQxf9i92435JGKN053Fe14AR70MtBSzIU2/zbgvhB1vF5lZt2i46hNxfo4lXR495gEwdwC6+8HJnlEJAJkdeHfZKj+RiwXoiYKPKsSbHi83X8Hh32IifuglNxdlxVQYAQ==", - "type": "thinking" - }, - { - "text": "I'll complete this instrumented research flow step by step. Starting with Step 1:", - "type": "text" - }, - { - "arguments": { - "query": "flue instrumentation" - }, - "id": "", - "name": "lookup", - "type": "toolCall" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "anthropic-messages", - "flue.input_mode": "full", - "flue.model": "claude-sonnet-4-5-20250929", - "flue.provider": "anthropic", - "flue.session": "main", - "flue.stop_reason": "toolUse", - "flue.turn_purpose": "agent", - "model": "claude-sonnet-4-5-20250929", - "provider": "anthropic" - }, - "metrics": { - "completion_tokens": 196, - "duration_ms": 0, - "estimated_cost": 0.0113025, - "prompt_cache_creation_tokens": 2222, - "prompt_cached_tokens": 0, - "prompt_tokens": 10, - "tokens": 2428 - } - }, - { - "name": "tool:lookup", - "type": "tool", - "children": [], - "input": { - "query": "flue instrumentation" - }, - "output": { - "content": [ - { - "text": "{\"id\":\"flue-session-2026\",\"query\":\"flue instrumentation\",\"topic\":\"session instrumentation\"}", - "type": "text" - } - ], - "details": { - "customTool": "lookup" - } - }, - "metadata": { - "flue.session": "main", - "flue.tool_name": "lookup", - "provider": "flue" - }, - "metrics": { - "duration_ms": 0 - } - }, - { - "name": "flue.turn", - "type": "llm", - "children": [], - "input": [ - { - "content": [ - { - "thinking": "The user wants me to complete a research flow with exactly three steps:\n1. Call lookup with query \"flue instrumentation\"\n2. Call web_search with the lookupId from step 1 and query \"Braintrust Flue reasoning stream instrumentation\"\n3. Call summarize_source with the first URL from step 2\n4. Reply with exactly \"PROMPT_DONE\"\n\nThe user emphasizes I should call exactly one tool per turn and wait for each result. Let me start with step 1.", - "thinkingSignature": "EoQFCm4IDhgCKkD+AoKMhpXNPUJ5eANwM28uUCJPCE+AXh0ACw7aXniX+YN04zdJl3hlHMbtAuKhhczLiEeOJtu4KtVpea5u5NFdMhpjbGF1ZGUtc29ubmV0LTQtNS0yMDI1MDkyOTgAQgh0aGlua2luZxIMIOTWCNEtsiI9vF4SGgwPXC861uZx8lpOziEiMLsCukGIluD0U44degzYnHFc+BwXNIEB+yatSHlZE+a5smr9crZcft3OLmsIdXuaFSrDAyQxWd9oqfJdnvza9eWIaOCeNkVCjrTc/wotBwh5Xp8m9k/WKt7P4zzUJBuLQ1f6ZnsdWkL8OiL/2E7Up55z2+UuVWsg1Y90/x27YRFdQjv3RnztYp4bZ5oIHAlTJoCzWiBYEh/Z8c2O+wy6MHo1cp0T57OroB/4HAqRZxrjTOXpKYLmYKXAY5tU8DTd3A0/FfEhg3oPSbOhX+112szFCpm3BRNJYooUUhwSu1hO4H/6r7d/Yp9XIM4tBfzuDx83CC/YetdOFgnYkdliw0xaUGaj4o37rWIRctWHAMdlUc4NDA0Ljc1uirHDXkN8EJwWSoIjPSBTA/p/wgLrvRV5Yy9Z0dMoWOpgYF6t1H/tNAyY8ml1CA7IWyd3jnfTZ1/G3Y0xFDwJDFlb4/QKc67w36SSCa+X9mMoTxD/DYN249lsII+XX3lQ7F1zJ07THOtCyRirCaRhmE/kGwl5tiB1QHrWa9bNHxQxf9i92435JGKN053Fe14AR70MtBSzIU2/zbgvhB1vF5lZt2i46hNxfo4lXR495gEwdwC6+8HJnlEJAJkdeHfZKj+RiwXoiYKPKsSbHi83X8Hh32IifuglNxdlxVQYAQ==", - "type": "thinking" - }, - { - "text": "I'll complete this instrumented research flow step by step. Starting with Step 1:", - "type": "text" - }, - { - "arguments": { - "query": "flue instrumentation" - }, - "id": "", - "name": "lookup", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "{\"id\":\"flue-session-2026\",\"query\":\"flue instrumentation\",\"topic\":\"session instrumentation\"}", - "type": "text" - } - ], - "isError": false, - "role": "toolResult", - "toolCallId": "", - "toolName": "lookup" - } - ], - "output": { - "content": [ - { - "thinking": "Good, I got the lookup result with id \"flue-session-2026\". Now I need to proceed to step 2: call web_search with this lookupId and the query \"Braintrust Flue reasoning stream instrumentation\".", - "thinkingSignature": "EpEDCm4IDhgCKkAMahve1kB1j4njp1ZSzO8eCK4bb5Ex7MB6tjdoOXfuUDRfkBqvIGdwVly8GiA8GGYC7Y/RVfREXF6A6OXHTwFyMhpjbGF1ZGUtc29ubmV0LTQtNS0yMDI1MDkyOTgAQgh0aGlua2luZxIMLw+fsa4yo7a7ZSoIGgyaTdvH020n2dQj9bUiMB4OokY29w+uZb4rqDnKDbLXUqUj6U0f1QCGQF2ZBYDDfDJyM5Jv1xY3rQiUUjkhBirQAS4XivjvMnWI5zHBmpI/u/pq4aGdrdb3vUsJ/ne6R1d80JA/INGF8vLkQcZ5xxEK0jU4yRE+ldc2uzavmtoxIKAFD6bqKks1Qv6u+TIZtuW7tUB/7jFxDw93KmJNruMIa8c1J6pPUyBy3xE7QwgmU0kDfIfdnljxLoEBiMwtnxjCPj9lT7HHz4FkrhbnQTV4MvAQohCLjaSbOTd6LdLxwtJPm9mc26ID4eYN1gCCzyAxGXhqLHlzc23PafvQjsZliWnLhaiNMnG0YEo78XkuJQsYAQ==", - "type": "thinking" - }, - { - "text": "Now proceeding to Step 2:", - "type": "text" - }, - { - "arguments": { - "lookupId": "flue-session-2026", - "query": "Braintrust Flue reasoning stream instrumentation" - }, - "id": "", - "name": "web_search", - "type": "toolCall" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "anthropic-messages", - "flue.input_message_offset": 1, - "flue.input_mode": "delta", - "flue.model": "claude-sonnet-4-5-20250929", - "flue.provider": "anthropic", - "flue.session": "main", - "flue.stop_reason": "toolUse", - "flue.turn_purpose": "agent", - "model": "claude-sonnet-4-5-20250929", - "provider": "anthropic" - }, - "metrics": { - "completion_tokens": 158, - "duration_ms": 0, - "estimated_cost": 0.0039606, - "prompt_cache_creation_tokens": 236, - "prompt_cached_tokens": 2222, - "prompt_tokens": 13, - "tokens": 2629 - } - }, - { - "name": "tool:web_search", - "type": "tool", - "children": [], - "input": { - "lookupId": "flue-session-2026", - "query": "Braintrust Flue reasoning stream instrumentation" - }, - "output": { - "content": [ - { - "text": "{\"lookupId\":\"flue-session-2026\",\"query\":\"Braintrust Flue reasoning stream instrumentation\",\"results\":[{\"title\":\"Flue reasoning stream instrumentation\",\"url\":\"https://example.test/flue/reasoning-streams\"}]}", - "type": "text" - } - ], - "details": { - "customTool": "web_search" - } - }, - "metadata": { - "flue.session": "main", - "flue.tool_name": "web_search", - "provider": "flue" - }, - "metrics": { - "duration_ms": 0 - } - }, - { - "name": "flue.turn", - "type": "llm", - "children": [], - "input": [ - { - "content": [ - { - "thinking": "Good, I got the lookup result with id \"flue-session-2026\". Now I need to proceed to step 2: call web_search with this lookupId and the query \"Braintrust Flue reasoning stream instrumentation\".", - "thinkingSignature": "EpEDCm4IDhgCKkAMahve1kB1j4njp1ZSzO8eCK4bb5Ex7MB6tjdoOXfuUDRfkBqvIGdwVly8GiA8GGYC7Y/RVfREXF6A6OXHTwFyMhpjbGF1ZGUtc29ubmV0LTQtNS0yMDI1MDkyOTgAQgh0aGlua2luZxIMLw+fsa4yo7a7ZSoIGgyaTdvH020n2dQj9bUiMB4OokY29w+uZb4rqDnKDbLXUqUj6U0f1QCGQF2ZBYDDfDJyM5Jv1xY3rQiUUjkhBirQAS4XivjvMnWI5zHBmpI/u/pq4aGdrdb3vUsJ/ne6R1d80JA/INGF8vLkQcZ5xxEK0jU4yRE+ldc2uzavmtoxIKAFD6bqKks1Qv6u+TIZtuW7tUB/7jFxDw93KmJNruMIa8c1J6pPUyBy3xE7QwgmU0kDfIfdnljxLoEBiMwtnxjCPj9lT7HHz4FkrhbnQTV4MvAQohCLjaSbOTd6LdLxwtJPm9mc26ID4eYN1gCCzyAxGXhqLHlzc23PafvQjsZliWnLhaiNMnG0YEo78XkuJQsYAQ==", - "type": "thinking" - }, - { - "text": "Now proceeding to Step 2:", - "type": "text" - }, - { - "arguments": { - "lookupId": "flue-session-2026", - "query": "Braintrust Flue reasoning stream instrumentation" - }, - "id": "", - "name": "web_search", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "{\"lookupId\":\"flue-session-2026\",\"query\":\"Braintrust Flue reasoning stream instrumentation\",\"results\":[{\"title\":\"Flue reasoning stream instrumentation\",\"url\":\"https://example.test/flue/reasoning-streams\"}]}", - "type": "text" - } - ], - "isError": false, - "role": "toolResult", - "toolCallId": "", - "toolName": "web_search" - } - ], - "output": { - "content": [ - { - "thinking": "Perfect! I got the web_search results with one result. The first (and only) result has the URL \"https://example.test/flue/reasoning-streams\". Now I need to proceed to step 3: call summarize_source with this URL.", - "thinkingSignature": "EqQDCm4IDhgCKkAXv7EDAhWMcrBz11bUd5qxlzyy03CAGQfRgLHDE0uKzLATh1o2sedd8GxLwwQqV0CmLRfNxsdK/ZDp3Lks9iysMhpjbGF1ZGUtc29ubmV0LTQtNS0yMDI1MDkyOTgAQgh0aGlua2luZxIMXDMniOy5Dy0KI8x7GgzgHs7BYJMs6/N0ur8iMPe0SwnHhP39OjEu7P8ASwK9ffwa3VP67QOlPzOstRRdEZNd0mq+BVjpmBXB+QcAzCrjAeAh/UO/d7SfFmXjqwA3//thoL/TTlGa6W+fdOCFRikn65JWT4ZdVFetfqD+9/QCP/nXEwMsLnTBAxC4EGryrgs2YT+OWwmEJ7rHKFeDs5djhwSb2J2ZJO1kRvD9s2vCCDkImygYBSIVgx1Y0CZJHhzn2Vf39v0Fbth2wDZa+OqxivV9NmDZEjmgcM+tTRx1CCnaww8LZEJQ68v/dUJy7hhD4MrNcQ62evJYPfT3+4EKKWJ/7HI7KXz91//+YAkph63XstbJMFJ3AldAFiq3ecauXt5r42N5gmCmwZX+sI48ENLGGAE=", - "type": "thinking" - }, - { - "text": "Now proceeding to Step 3:", - "type": "text" - }, - { - "arguments": { - "url": "https://example.test/flue/reasoning-streams" - }, - "id": "", - "name": "summarize_source", - "type": "toolCall" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "anthropic-messages", - "flue.input_message_offset": 3, - "flue.input_mode": "delta", - "flue.model": "claude-sonnet-4-5-20250929", - "flue.provider": "anthropic", - "flue.session": "main", - "flue.stop_reason": "toolUse", - "flue.turn_purpose": "agent", - "model": "claude-sonnet-4-5-20250929", - "provider": "anthropic" - }, - "metrics": { - "completion_tokens": 138, - "duration_ms": 0, - "estimated_cost": 0.0037239000000000005, - "prompt_cache_creation_tokens": 234, - "prompt_cached_tokens": 2458, - "prompt_tokens": 13, - "tokens": 2843 - } - }, - { - "name": "tool:summarize_source", - "type": "tool", - "children": [], - "input": { - "url": "https://example.test/flue/reasoning-streams" - }, - "output": { - "content": [ - { - "text": "{\"summary\":\"Flue emits reasoning, tool execution, and LLM turn events separately.\",\"url\":\"https://example.test/flue/reasoning-streams\"}", - "type": "text" - } - ], - "details": { - "customTool": "summarize_source" - } - }, - "metadata": { - "flue.session": "main", - "flue.tool_name": "summarize_source", - "provider": "flue" - }, - "metrics": { - "duration_ms": 0 - } - }, - { - "name": "flue.turn", - "type": "llm", - "children": [], - "input": [ - { - "content": [ - { - "thinking": "Perfect! I got the web_search results with one result. The first (and only) result has the URL \"https://example.test/flue/reasoning-streams\". Now I need to proceed to step 3: call summarize_source with this URL.", - "thinkingSignature": "EqQDCm4IDhgCKkAXv7EDAhWMcrBz11bUd5qxlzyy03CAGQfRgLHDE0uKzLATh1o2sedd8GxLwwQqV0CmLRfNxsdK/ZDp3Lks9iysMhpjbGF1ZGUtc29ubmV0LTQtNS0yMDI1MDkyOTgAQgh0aGlua2luZxIMXDMniOy5Dy0KI8x7GgzgHs7BYJMs6/N0ur8iMPe0SwnHhP39OjEu7P8ASwK9ffwa3VP67QOlPzOstRRdEZNd0mq+BVjpmBXB+QcAzCrjAeAh/UO/d7SfFmXjqwA3//thoL/TTlGa6W+fdOCFRikn65JWT4ZdVFetfqD+9/QCP/nXEwMsLnTBAxC4EGryrgs2YT+OWwmEJ7rHKFeDs5djhwSb2J2ZJO1kRvD9s2vCCDkImygYBSIVgx1Y0CZJHhzn2Vf39v0Fbth2wDZa+OqxivV9NmDZEjmgcM+tTRx1CCnaww8LZEJQ68v/dUJy7hhD4MrNcQ62evJYPfT3+4EKKWJ/7HI7KXz91//+YAkph63XstbJMFJ3AldAFiq3ecauXt5r42N5gmCmwZX+sI48ENLGGAE=", - "type": "thinking" - }, - { - "text": "Now proceeding to Step 3:", - "type": "text" - }, - { - "arguments": { - "url": "https://example.test/flue/reasoning-streams" - }, - "id": "", - "name": "summarize_source", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "{\"summary\":\"Flue emits reasoning, tool execution, and LLM turn events separately.\",\"url\":\"https://example.test/flue/reasoning-streams\"}", - "type": "text" - } - ], - "isError": false, - "role": "toolResult", - "toolCallId": "", - "toolName": "summarize_source" - } - ], - "output": { - "content": [ - { - "thinking": "Perfect! The summarize_source call has completed successfully. According to the instructions, I should now reply with exactly \"PROMPT_DONE\" and no other text.", - "thinkingSignature": "Eu8CCm4IDhgCKkAn2QOIIlYfvcs6ynsmhCslHDtBsrTA4CfmimK7zyz6+k6dZldV2JBtEBET41It0gFcGEKC0n0mAhaYVuLutpDDMhpjbGF1ZGUtc29ubmV0LTQtNS0yMDI1MDkyOTgAQgh0aGlua2luZxIM1xfLptxSF8rKzou8GgwRLuVmfiBYOT4CZIsiMAYz8TpvchMAWodP+T9uZ+dJ47MVdNv+p86nZXmT5d5EWlCroiAOFHHzJH2o0UjJQSquAX6wu+jIC8/DXwurGIIf2ToldPuJWF5HGj8kHMnwh777XS1NUFl1irM24rvP7kTAv3Dp8YEwb8t3g1EsicY7j+C5DzgwxVsPYCVOCgdMqQjtsqpL7AX5KQ9+hR1gK1DQjshv4Gze3hw/+C+9iptd3RlDIuwO0cm8NY+sVqeWVh3TtQujQlVt1rW/HfWjclrtT0LZSGwc6RypmfMMQi6lncIk/EPgy0XbS1plKW0tLBgB", - "type": "thinking" - }, - { - "text": "PROMPT_DONE", - "type": "text" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "anthropic-messages", - "flue.input_message_offset": 5, - "flue.input_mode": "delta", - "flue.model": "claude-sonnet-4-5-20250929", - "flue.provider": "anthropic", - "flue.session": "main", - "flue.stop_reason": "stop", - "flue.turn_purpose": "agent", - "model": "claude-sonnet-4-5-20250929", - "provider": "anthropic" - }, - "metrics": { - "completion_tokens": 51, - "duration_ms": 0, - "estimated_cost": 0.0023391, - "prompt_cache_creation_tokens": 194, - "prompt_cached_tokens": 2692, - "prompt_tokens": 13, - "tokens": 2950 - } - } - ], - "input": [ - { - "content": [ - { - "text": "Complete this instrumented research flow. Call exactly one tool per turn and wait for each tool result before choosing the next tool. Step 1: call lookup with query \"flue instrumentation\". Step 2: use the lookup result id as lookupId and call web_search with query \"Braintrust Flue reasoning stream instrumentation\". Step 3: use the first web_search result url and call summarize_source. After summarize_source returns, reply with exactly PROMPT_DONE and no other text.", - "type": "text" - } - ], - "role": "user" - } - ], - "output": "PROMPT_DONE", - "metadata": { - "flue.operation": "prompt", - "flue.session": "main", - "flue.workflow_name": "instrumentation", - "provider": "flue", - "scenario": "flue-instrumentation" - }, - "metrics": { - "duration_ms": 0 - } - } - ] -} diff --git a/e2e/scenarios/flue-instrumentation/__snapshots__/flue-v0-8-0-explicit.span-tree.txt b/e2e/scenarios/flue-instrumentation/__snapshots__/flue-v0-8-0-explicit.span-tree.txt deleted file mode 100644 index e90a4892a..000000000 --- a/e2e/scenarios/flue-instrumentation/__snapshots__/flue-v0-8-0-explicit.span-tree.txt +++ /dev/null @@ -1,1335 +0,0 @@ -span_tree: -├── workflow:instrumentation [task] -│ input: { -│ "metadata": { -│ "scenario": "flue-instrumentation", -│ "testRunId": "" -│ }, -│ "scenario": "flue-instrumentation" -│ } -│ output: { -│ "scenario": "flue-instrumentation", -│ "status": "done" -│ } -│ metadata: { -│ "flue.workflow_name": "instrumentation", -│ "provider": "flue", -│ "scenario": "flue-instrumentation" -│ } -│ metrics: { -│ "duration_ms": 0 -│ } -│ ├── flue.skill [task] -│ │ input: [ -│ │ { -│ │ "content": [ -│ │ { -│ │ "text": "Run the skill named \"e2e-flue-skill\".\n\nArguments:\n{\n \"marker\": \"SKILL_DONE\"\n}", -│ │ "type": "text" -│ │ } -│ │ ], -│ │ "role": "user" -│ │ } -│ │ ] -│ │ output: "SKILL_DONE" -│ │ metadata: { -│ │ "flue.operation": "skill", -│ │ "flue.session": "skill", -│ │ "flue.workflow_name": "instrumentation", -│ │ "provider": "flue", -│ │ "scenario": "flue-instrumentation" -│ │ } -│ │ metrics: { -│ │ "duration_ms": 0 -│ │ } -│ │ ├── flue.turn [llm] -│ │ │ input: [ -│ │ │ { -│ │ │ "content": [ -│ │ │ { -│ │ │ "text": "Run the skill named \"e2e-flue-skill\".\n\nArguments:\n{\n \"marker\": \"SKILL_DONE\"\n}", -│ │ │ "type": "text" -│ │ │ } -│ │ │ ], -│ │ │ "role": "user" -│ │ │ } -│ │ │ ] -│ │ │ output: { -│ │ │ "content": [ -│ │ │ { -│ │ │ "text": "I'll run the e2e-flue-skill. Let me first read the skill file to understand its instructions.", -│ │ │ "type": "text" -│ │ │ }, -│ │ │ { -│ │ │ "arguments": { -│ │ │ "path": ".agents/e2e-flue-skill.md" -│ │ │ }, -│ │ │ "id": "", -│ │ │ "name": "read", -│ │ │ "type": "toolCall" -│ │ │ } -│ │ │ ], -│ │ │ "role": "assistant" -│ │ │ } -│ │ │ metadata: { -│ │ │ "flue.api": "anthropic-messages", -│ │ │ "flue.model": "claude-sonnet-4-5-20250929", -│ │ │ "flue.provider": "anthropic", -│ │ │ "flue.session": "skill", -│ │ │ "flue.stop_reason": "toolUse", -│ │ │ "flue.turn_purpose": "agent", -│ │ │ "model": "claude-sonnet-4-5-20250929", -│ │ │ "provider": "anthropic" -│ │ │ } -│ │ │ metrics: { -│ │ │ "completion_tokens": 89, -│ │ │ "duration_ms": 0, -│ │ │ "estimated_cost": 0.0041862, -│ │ │ "prompt_cache_creation_tokens": 670, -│ │ │ "prompt_cached_tokens": 1099, -│ │ │ "prompt_tokens": 3, -│ │ │ "tokens": 1861 -│ │ │ } -│ │ ├── tool:read [tool] -│ │ │ input: { -│ │ │ "path": ".agents/e2e-flue-skill.md" -│ │ │ } -│ │ │ output: { -│ │ │ "content": [ -│ │ │ { -│ │ │ "text": "ENOENT: no such file or directory, open '/e2e/scenarios/flue-instrumentation/.agents/e2e-flue-skill.md'", -│ │ │ "type": "text" -│ │ │ } -│ │ │ ], -│ │ │ "details": {} -│ │ │ } -│ │ │ metadata: { -│ │ │ "flue.session": "skill", -│ │ │ "flue.tool_name": "read", -│ │ │ "provider": "flue" -│ │ │ } -│ │ │ metrics: { -│ │ │ "duration_ms": 0 -│ │ │ } -│ │ │ error: "{\"content\":[{\"type\":\"text\",\"text\":\"ENOENT: no such file or directory, open '/e2e/scenarios/flue-instrumentation/.agents/e2e-flue-skill.md'\"}],\"details\":{}}" -│ │ ├── flue.turn [llm] -│ │ │ input: [ -│ │ │ { -│ │ │ "content": [ -│ │ │ { -│ │ │ "text": "Run the skill named \"e2e-flue-skill\".\n\nArguments:\n{\n \"marker\": \"SKILL_DONE\"\n}", -│ │ │ "type": "text" -│ │ │ } -│ │ │ ], -│ │ │ "role": "user" -│ │ │ }, -│ │ │ { -│ │ │ "content": [ -│ │ │ { -│ │ │ "text": "I'll run the e2e-flue-skill. Let me first read the skill file to understand its instructions.", -│ │ │ "type": "text" -│ │ │ }, -│ │ │ { -│ │ │ "arguments": { -│ │ │ "path": ".agents/e2e-flue-skill.md" -│ │ │ }, -│ │ │ "id": "", -│ │ │ "name": "read", -│ │ │ "type": "toolCall" -│ │ │ } -│ │ │ ], -│ │ │ "role": "assistant" -│ │ │ }, -│ │ │ { -│ │ │ "content": [ -│ │ │ { -│ │ │ "text": "ENOENT: no such file or directory, open '/e2e/scenarios/flue-instrumentation/.agents/e2e-flue-skill.md'", -│ │ │ "type": "text" -│ │ │ } -│ │ │ ], -│ │ │ "isError": true, -│ │ │ "role": "toolResult", -│ │ │ "toolCallId": "", -│ │ │ "toolName": "read" -│ │ │ } -│ │ │ ] -│ │ │ output: { -│ │ │ "content": [ -│ │ │ { -│ │ │ "arguments": { -│ │ │ "path": ".agents" -│ │ │ }, -│ │ │ "id": "", -│ │ │ "name": "read", -│ │ │ "type": "toolCall" -│ │ │ } -│ │ │ ], -│ │ │ "role": "assistant" -│ │ │ } -│ │ │ metadata: { -│ │ │ "flue.api": "anthropic-messages", -│ │ │ "flue.model": "claude-sonnet-4-5-20250929", -│ │ │ "flue.provider": "anthropic", -│ │ │ "flue.session": "skill", -│ │ │ "flue.stop_reason": "toolUse", -│ │ │ "flue.turn_purpose": "agent", -│ │ │ "model": "claude-sonnet-4-5-20250929", -│ │ │ "provider": "anthropic" -│ │ │ } -│ │ │ metrics: { -│ │ │ "completion_tokens": 52, -│ │ │ "duration_ms": 0, -│ │ │ "estimated_cost": 0.00201795, -│ │ │ "prompt_cache_creation_tokens": 183, -│ │ │ "prompt_cached_tokens": 1769, -│ │ │ "prompt_tokens": 7, -│ │ │ "tokens": 2011 -│ │ │ } -│ │ ├── tool:read [tool] -│ │ │ input: { -│ │ │ "path": ".agents" -│ │ │ } -│ │ │ output: { -│ │ │ "content": [ -│ │ │ { -│ │ │ "text": "skills", -│ │ │ "type": "text" -│ │ │ } -│ │ │ ], -│ │ │ "details": { -│ │ │ "entries": 1, -│ │ │ "isDirectory": true, -│ │ │ "path": ".agents" -│ │ │ } -│ │ │ } -│ │ │ metadata: { -│ │ │ "flue.session": "skill", -│ │ │ "flue.tool_name": "read", -│ │ │ "provider": "flue" -│ │ │ } -│ │ │ metrics: { -│ │ │ "duration_ms": 0 -│ │ │ } -│ │ ├── flue.turn [llm] -│ │ │ input: [ -│ │ │ { -│ │ │ "content": [ -│ │ │ { -│ │ │ "text": "Run the skill named \"e2e-flue-skill\".\n\nArguments:\n{\n \"marker\": \"SKILL_DONE\"\n}", -│ │ │ "type": "text" -│ │ │ } -│ │ │ ], -│ │ │ "role": "user" -│ │ │ }, -│ │ │ { -│ │ │ "content": [ -│ │ │ { -│ │ │ "text": "I'll run the e2e-flue-skill. Let me first read the skill file to understand its instructions.", -│ │ │ "type": "text" -│ │ │ }, -│ │ │ { -│ │ │ "arguments": { -│ │ │ "path": ".agents/e2e-flue-skill.md" -│ │ │ }, -│ │ │ "id": "", -│ │ │ "name": "read", -│ │ │ "type": "toolCall" -│ │ │ } -│ │ │ ], -│ │ │ "role": "assistant" -│ │ │ }, -│ │ │ { -│ │ │ "content": [ -│ │ │ { -│ │ │ "text": "ENOENT: no such file or directory, open '/e2e/scenarios/flue-instrumentation/.agents/e2e-flue-skill.md'", -│ │ │ "type": "text" -│ │ │ } -│ │ │ ], -│ │ │ "isError": true, -│ │ │ "role": "toolResult", -│ │ │ "toolCallId": "", -│ │ │ "toolName": "read" -│ │ │ }, -│ │ │ { -│ │ │ "content": [ -│ │ │ { -│ │ │ "arguments": { -│ │ │ "path": ".agents" -│ │ │ }, -│ │ │ "id": "", -│ │ │ "name": "read", -│ │ │ "type": "toolCall" -│ │ │ } -│ │ │ ], -│ │ │ "role": "assistant" -│ │ │ }, -│ │ │ { -│ │ │ "content": [ -│ │ │ { -│ │ │ "text": "skills", -│ │ │ "type": "text" -│ │ │ } -│ │ │ ], -│ │ │ "isError": false, -│ │ │ "role": "toolResult", -│ │ │ "toolCallId": "", -│ │ │ "toolName": "read" -│ │ │ } -│ │ │ ] -│ │ │ output: { -│ │ │ "content": [ -│ │ │ { -│ │ │ "arguments": { -│ │ │ "path": ".agents/skills" -│ │ │ }, -│ │ │ "id": "", -│ │ │ "name": "read", -│ │ │ "type": "toolCall" -│ │ │ } -│ │ │ ], -│ │ │ "role": "assistant" -│ │ │ } -│ │ │ metadata: { -│ │ │ "flue.api": "anthropic-messages", -│ │ │ "flue.model": "claude-sonnet-4-5-20250929", -│ │ │ "flue.provider": "anthropic", -│ │ │ "flue.session": "skill", -│ │ │ "flue.stop_reason": "toolUse", -│ │ │ "flue.turn_purpose": "agent", -│ │ │ "model": "claude-sonnet-4-5-20250929", -│ │ │ "provider": "anthropic" -│ │ │ } -│ │ │ metrics: { -│ │ │ "completion_tokens": 54, -│ │ │ "duration_ms": 0, -│ │ │ "estimated_cost": 0.0016611, -│ │ │ "prompt_cache_creation_tokens": 66, -│ │ │ "prompt_cached_tokens": 1952, -│ │ │ "prompt_tokens": 6, -│ │ │ "tokens": 2078 -│ │ │ } -│ │ ├── tool:read [tool] -│ │ │ input: { -│ │ │ "path": ".agents/skills" -│ │ │ } -│ │ │ output: { -│ │ │ "content": [ -│ │ │ { -│ │ │ "text": "e2e-flue-skill", -│ │ │ "type": "text" -│ │ │ } -│ │ │ ], -│ │ │ "details": { -│ │ │ "entries": 1, -│ │ │ "isDirectory": true, -│ │ │ "path": ".agents/skills" -│ │ │ } -│ │ │ } -│ │ │ metadata: { -│ │ │ "flue.session": "skill", -│ │ │ "flue.tool_name": "read", -│ │ │ "provider": "flue" -│ │ │ } -│ │ │ metrics: { -│ │ │ "duration_ms": 0 -│ │ │ } -│ │ ├── flue.turn [llm] -│ │ │ input: [ -│ │ │ { -│ │ │ "content": [ -│ │ │ { -│ │ │ "text": "Run the skill named \"e2e-flue-skill\".\n\nArguments:\n{\n \"marker\": \"SKILL_DONE\"\n}", -│ │ │ "type": "text" -│ │ │ } -│ │ │ ], -│ │ │ "role": "user" -│ │ │ }, -│ │ │ { -│ │ │ "content": [ -│ │ │ { -│ │ │ "text": "I'll run the e2e-flue-skill. Let me first read the skill file to understand its instructions.", -│ │ │ "type": "text" -│ │ │ }, -│ │ │ { -│ │ │ "arguments": { -│ │ │ "path": ".agents/e2e-flue-skill.md" -│ │ │ }, -│ │ │ "id": "", -│ │ │ "name": "read", -│ │ │ "type": "toolCall" -│ │ │ } -│ │ │ ], -│ │ │ "role": "assistant" -│ │ │ }, -│ │ │ { -│ │ │ "content": [ -│ │ │ { -│ │ │ "text": "ENOENT: no such file or directory, open '/e2e/scenarios/flue-instrumentation/.agents/e2e-flue-skill.md'", -│ │ │ "type": "text" -│ │ │ } -│ │ │ ], -│ │ │ "isError": true, -│ │ │ "role": "toolResult", -│ │ │ "toolCallId": "", -│ │ │ "toolName": "read" -│ │ │ }, -│ │ │ { -│ │ │ "content": [ -│ │ │ { -│ │ │ "arguments": { -│ │ │ "path": ".agents" -│ │ │ }, -│ │ │ "id": "", -│ │ │ "name": "read", -│ │ │ "type": "toolCall" -│ │ │ } -│ │ │ ], -│ │ │ "role": "assistant" -│ │ │ }, -│ │ │ { -│ │ │ "content": [ -│ │ │ { -│ │ │ "text": "skills", -│ │ │ "type": "text" -│ │ │ } -│ │ │ ], -│ │ │ "isError": false, -│ │ │ "role": "toolResult", -│ │ │ "toolCallId": "", -│ │ │ "toolName": "read" -│ │ │ }, -│ │ │ { -│ │ │ "content": [ -│ │ │ { -│ │ │ "arguments": { -│ │ │ "path": ".agents/skills" -│ │ │ }, -│ │ │ "id": "", -│ │ │ "name": "read", -│ │ │ "type": "toolCall" -│ │ │ } -│ │ │ ], -│ │ │ "role": "assistant" -│ │ │ }, -│ │ │ { -│ │ │ "content": [ -│ │ │ { -│ │ │ "text": "e2e-flue-skill", -│ │ │ "type": "text" -│ │ │ } -│ │ │ ], -│ │ │ "isError": false, -│ │ │ "role": "toolResult", -│ │ │ "toolCallId": "", -│ │ │ "toolName": "read" -│ │ │ } -│ │ │ ] -│ │ │ output: { -│ │ │ "content": [ -│ │ │ { -│ │ │ "arguments": { -│ │ │ "path": ".agents/skills/e2e-flue-skill" -│ │ │ }, -│ │ │ "id": "", -│ │ │ "name": "read", -│ │ │ "type": "toolCall" -│ │ │ } -│ │ │ ], -│ │ │ "role": "assistant" -│ │ │ } -│ │ │ metadata: { -│ │ │ "flue.api": "anthropic-messages", -│ │ │ "flue.model": "claude-sonnet-4-5-20250929", -│ │ │ "flue.provider": "anthropic", -│ │ │ "flue.session": "skill", -│ │ │ "flue.stop_reason": "toolUse", -│ │ │ "flue.turn_purpose": "agent", -│ │ │ "model": "claude-sonnet-4-5-20250929", -│ │ │ "provider": "anthropic" -│ │ │ } -│ │ │ metrics: { -│ │ │ "completion_tokens": 63, -│ │ │ "duration_ms": 0, -│ │ │ "estimated_cost": 0.0018459, -│ │ │ "prompt_cache_creation_tokens": 74, -│ │ │ "prompt_cached_tokens": 2018, -│ │ │ "prompt_tokens": 6, -│ │ │ "tokens": 2161 -│ │ │ } -│ │ ├── tool:read [tool] -│ │ │ input: { -│ │ │ "path": ".agents/skills/e2e-flue-skill" -│ │ │ } -│ │ │ output: { -│ │ │ "content": [ -│ │ │ { -│ │ │ "text": "SKILL.md", -│ │ │ "type": "text" -│ │ │ } -│ │ │ ], -│ │ │ "details": { -│ │ │ "entries": 1, -│ │ │ "isDirectory": true, -│ │ │ "path": ".agents/skills/e2e-flue-skill" -│ │ │ } -│ │ │ } -│ │ │ metadata: { -│ │ │ "flue.session": "skill", -│ │ │ "flue.tool_name": "read", -│ │ │ "provider": "flue" -│ │ │ } -│ │ │ metrics: { -│ │ │ "duration_ms": 0 -│ │ │ } -│ │ ├── flue.turn [llm] -│ │ │ input: [ -│ │ │ { -│ │ │ "content": [ -│ │ │ { -│ │ │ "text": "Run the skill named \"e2e-flue-skill\".\n\nArguments:\n{\n \"marker\": \"SKILL_DONE\"\n}", -│ │ │ "type": "text" -│ │ │ } -│ │ │ ], -│ │ │ "role": "user" -│ │ │ }, -│ │ │ { -│ │ │ "content": [ -│ │ │ { -│ │ │ "text": "I'll run the e2e-flue-skill. Let me first read the skill file to understand its instructions.", -│ │ │ "type": "text" -│ │ │ }, -│ │ │ { -│ │ │ "arguments": { -│ │ │ "path": ".agents/e2e-flue-skill.md" -│ │ │ }, -│ │ │ "id": "", -│ │ │ "name": "read", -│ │ │ "type": "toolCall" -│ │ │ } -│ │ │ ], -│ │ │ "role": "assistant" -│ │ │ }, -│ │ │ { -│ │ │ "content": [ -│ │ │ { -│ │ │ "text": "ENOENT: no such file or directory, open '/e2e/scenarios/flue-instrumentation/.agents/e2e-flue-skill.md'", -│ │ │ "type": "text" -│ │ │ } -│ │ │ ], -│ │ │ "isError": true, -│ │ │ "role": "toolResult", -│ │ │ "toolCallId": "", -│ │ │ "toolName": "read" -│ │ │ }, -│ │ │ { -│ │ │ "content": [ -│ │ │ { -│ │ │ "arguments": { -│ │ │ "path": ".agents" -│ │ │ }, -│ │ │ "id": "", -│ │ │ "name": "read", -│ │ │ "type": "toolCall" -│ │ │ } -│ │ │ ], -│ │ │ "role": "assistant" -│ │ │ }, -│ │ │ { -│ │ │ "content": [ -│ │ │ { -│ │ │ "text": "skills", -│ │ │ "type": "text" -│ │ │ } -│ │ │ ], -│ │ │ "isError": false, -│ │ │ "role": "toolResult", -│ │ │ "toolCallId": "", -│ │ │ "toolName": "read" -│ │ │ }, -│ │ │ { -│ │ │ "content": [ -│ │ │ { -│ │ │ "arguments": { -│ │ │ "path": ".agents/skills" -│ │ │ }, -│ │ │ "id": "", -│ │ │ "name": "read", -│ │ │ "type": "toolCall" -│ │ │ } -│ │ │ ], -│ │ │ "role": "assistant" -│ │ │ }, -│ │ │ { -│ │ │ "content": [ -│ │ │ { -│ │ │ "text": "e2e-flue-skill", -│ │ │ "type": "text" -│ │ │ } -│ │ │ ], -│ │ │ "isError": false, -│ │ │ "role": "toolResult", -│ │ │ "toolCallId": "", -│ │ │ "toolName": "read" -│ │ │ }, -│ │ │ { -│ │ │ "content": [ -│ │ │ { -│ │ │ "arguments": { -│ │ │ "path": ".agents/skills/e2e-flue-skill" -│ │ │ }, -│ │ │ "id": "", -│ │ │ "name": "read", -│ │ │ "type": "toolCall" -│ │ │ } -│ │ │ ], -│ │ │ "role": "assistant" -│ │ │ }, -│ │ │ { -│ │ │ "content": [ -│ │ │ { -│ │ │ "text": "SKILL.md", -│ │ │ "type": "text" -│ │ │ } -│ │ │ ], -│ │ │ "isError": false, -│ │ │ "role": "toolResult", -│ │ │ "toolCallId": "", -│ │ │ "toolName": "read" -│ │ │ } -│ │ │ ] -│ │ │ output: { -│ │ │ "content": [ -│ │ │ { -│ │ │ "arguments": { -│ │ │ "path": ".agents/skills/e2e-flue-skill/SKILL.md" -│ │ │ }, -│ │ │ "id": "", -│ │ │ "name": "read", -│ │ │ "type": "toolCall" -│ │ │ } -│ │ │ ], -│ │ │ "role": "assistant" -│ │ │ } -│ │ │ metadata: { -│ │ │ "flue.api": "anthropic-messages", -│ │ │ "flue.model": "claude-sonnet-4-5-20250929", -│ │ │ "flue.provider": "anthropic", -│ │ │ "flue.session": "skill", -│ │ │ "flue.stop_reason": "toolUse", -│ │ │ "flue.turn_purpose": "agent", -│ │ │ "model": "claude-sonnet-4-5-20250929", -│ │ │ "provider": "anthropic" -│ │ │ } -│ │ │ metrics: { -│ │ │ "completion_tokens": 68, -│ │ │ "duration_ms": 0, -│ │ │ "estimated_cost": 0.0019618500000000002, -│ │ │ "prompt_cache_creation_tokens": 79, -│ │ │ "prompt_cached_tokens": 2092, -│ │ │ "prompt_tokens": 6, -│ │ │ "tokens": 2245 -│ │ │ } -│ │ ├── tool:read [tool] -│ │ │ input: { -│ │ │ "path": ".agents/skills/e2e-flue-skill/SKILL.md" -│ │ │ } -│ │ │ output: { -│ │ │ "content": [ -│ │ │ { -│ │ │ "text": "---\nname: e2e-flue-skill\ndescription: Deterministic skill used by the Braintrust Flue instrumentation e2e test.\n---\n\nReturn the marker from the provided args exactly once. Output no other text.\n", -│ │ │ "type": "text" -│ │ │ } -│ │ │ ], -│ │ │ "details": { -│ │ │ "lines": 7, -│ │ │ "path": ".agents/skills/e2e-flue-skill/SKILL.md" -│ │ │ } -│ │ │ } -│ │ │ metadata: { -│ │ │ "flue.session": "skill", -│ │ │ "flue.tool_name": "read", -│ │ │ "provider": "flue" -│ │ │ } -│ │ │ metrics: { -│ │ │ "duration_ms": 0 -│ │ │ } -│ │ └── flue.turn [llm] -│ │ input: [ -│ │ { -│ │ "content": [ -│ │ { -│ │ "text": "Run the skill named \"e2e-flue-skill\".\n\nArguments:\n{\n \"marker\": \"SKILL_DONE\"\n}", -│ │ "type": "text" -│ │ } -│ │ ], -│ │ "role": "user" -│ │ }, -│ │ { -│ │ "content": [ -│ │ { -│ │ "text": "I'll run the e2e-flue-skill. Let me first read the skill file to understand its instructions.", -│ │ "type": "text" -│ │ }, -│ │ { -│ │ "arguments": { -│ │ "path": ".agents/e2e-flue-skill.md" -│ │ }, -│ │ "id": "", -│ │ "name": "read", -│ │ "type": "toolCall" -│ │ } -│ │ ], -│ │ "role": "assistant" -│ │ }, -│ │ { -│ │ "content": [ -│ │ { -│ │ "text": "ENOENT: no such file or directory, open '/e2e/scenarios/flue-instrumentation/.agents/e2e-flue-skill.md'", -│ │ "type": "text" -│ │ } -│ │ ], -│ │ "isError": true, -│ │ "role": "toolResult", -│ │ "toolCallId": "", -│ │ "toolName": "read" -│ │ }, -│ │ { -│ │ "content": [ -│ │ { -│ │ "arguments": { -│ │ "path": ".agents" -│ │ }, -│ │ "id": "", -│ │ "name": "read", -│ │ "type": "toolCall" -│ │ } -│ │ ], -│ │ "role": "assistant" -│ │ }, -│ │ { -│ │ "content": [ -│ │ { -│ │ "text": "skills", -│ │ "type": "text" -│ │ } -│ │ ], -│ │ "isError": false, -│ │ "role": "toolResult", -│ │ "toolCallId": "", -│ │ "toolName": "read" -│ │ }, -│ │ { -│ │ "content": [ -│ │ { -│ │ "arguments": { -│ │ "path": ".agents/skills" -│ │ }, -│ │ "id": "", -│ │ "name": "read", -│ │ "type": "toolCall" -│ │ } -│ │ ], -│ │ "role": "assistant" -│ │ }, -│ │ { -│ │ "content": [ -│ │ { -│ │ "text": "e2e-flue-skill", -│ │ "type": "text" -│ │ } -│ │ ], -│ │ "isError": false, -│ │ "role": "toolResult", -│ │ "toolCallId": "", -│ │ "toolName": "read" -│ │ }, -│ │ { -│ │ "content": [ -│ │ { -│ │ "arguments": { -│ │ "path": ".agents/skills/e2e-flue-skill" -│ │ }, -│ │ "id": "", -│ │ "name": "read", -│ │ "type": "toolCall" -│ │ } -│ │ ], -│ │ "role": "assistant" -│ │ }, -│ │ { -│ │ "content": [ -│ │ { -│ │ "text": "SKILL.md", -│ │ "type": "text" -│ │ } -│ │ ], -│ │ "isError": false, -│ │ "role": "toolResult", -│ │ "toolCallId": "", -│ │ "toolName": "read" -│ │ }, -│ │ { -│ │ "content": [ -│ │ { -│ │ "arguments": { -│ │ "path": ".agents/skills/e2e-flue-skill/SKILL.md" -│ │ }, -│ │ "id": "", -│ │ "name": "read", -│ │ "type": "toolCall" -│ │ } -│ │ ], -│ │ "role": "assistant" -│ │ }, -│ │ { -│ │ "content": [ -│ │ { -│ │ "text": "---\nname: e2e-flue-skill\ndescription: Deterministic skill used by the Braintrust Flue instrumentation e2e test.\n---\n\nReturn the marker from the provided args exactly once. Output no other text.\n", -│ │ "type": "text" -│ │ } -│ │ ], -│ │ "isError": false, -│ │ "role": "toolResult", -│ │ "toolCallId": "", -│ │ "toolName": "read" -│ │ } -│ │ ] -│ │ output: { -│ │ "content": [ -│ │ { -│ │ "text": "SKILL_DONE", -│ │ "type": "text" -│ │ } -│ │ ], -│ │ "role": "assistant" -│ │ } -│ │ metadata: { -│ │ "flue.api": "anthropic-messages", -│ │ "flue.model": "claude-sonnet-4-5-20250929", -│ │ "flue.provider": "anthropic", -│ │ "flue.session": "skill", -│ │ "flue.stop_reason": "stop", -│ │ "flue.turn_purpose": "agent", -│ │ "model": "claude-sonnet-4-5-20250929", -│ │ "provider": "anthropic" -│ │ } -│ │ metrics: { -│ │ "completion_tokens": 8, -│ │ "duration_ms": 0, -│ │ "estimated_cost": 0.00129555, -│ │ "prompt_cache_creation_tokens": 135, -│ │ "prompt_cached_tokens": 2171, -│ │ "prompt_tokens": 6, -│ │ "tokens": 2320 -│ │ } -│ ├── flue.task [task] -│ │ input: "Reply with exactly TASK_DONE and no other text." -│ │ output: "TASK_DONE" -│ │ metadata: { -│ │ "flue.session": "task", -│ │ "provider": "flue" -│ │ } -│ │ metrics: { -│ │ "duration_ms": 0 -│ │ } -│ │ └── flue.turn [llm] -│ │ input: [ -│ │ { -│ │ "content": [ -│ │ { -│ │ "text": "Reply with exactly TASK_DONE and no other text.", -│ │ "type": "text" -│ │ } -│ │ ], -│ │ "role": "user" -│ │ } -│ │ ] -│ │ output: { -│ │ "content": [ -│ │ { -│ │ "text": "TASK_DONE", -│ │ "textSignature": "{\"v\":1,\"id\":\"msg_09b32b2821e9a4d8016a184600019081929d34bb7d4c6a246f\"}", -│ │ "type": "text" -│ │ } -│ │ ], -│ │ "role": "assistant" -│ │ } -│ │ metadata: { -│ │ "flue.api": "openai-responses", -│ │ "flue.input_mode": "full", -│ │ "flue.model": "gpt-4o-mini", -│ │ "flue.provider": "openai", -│ │ "flue.session": "task:task:", -│ │ "flue.stop_reason": "stop", -│ │ "flue.turn_purpose": "agent", -│ │ "model": "gpt-4o-mini", -│ │ "provider": "openai" -│ │ } -│ │ metrics: { -│ │ "completion_tokens": 4, -│ │ "duration_ms": 0, -│ │ "estimated_cost": 0.00012299999999999998, -│ │ "prompt_cache_creation_tokens": 0, -│ │ "prompt_cached_tokens": 0, -│ │ "prompt_tokens": 804, -│ │ "tokens": 808 -│ │ } -│ └── flue.compact [task] -│ input: { -│ "estimatedTokens": 2950, -│ "reason": "manual" -│ } -│ output: { -│ "completed": true -│ } -│ metadata: { -│ "flue.operation": "compact", -│ "flue.session": "main", -│ "flue.workflow_name": "instrumentation", -│ "provider": "flue", -│ "scenario": "flue-instrumentation" -│ } -│ metrics: { -│ "duration_ms": 0 -│ } -│ └── compaction:manual [task] -│ input: { -│ "estimatedTokens": 2950, -│ "reason": "manual" -│ } -│ output: { -│ "messagesAfter": 2, -│ "messagesBefore": 8 -│ } -│ metadata: { -│ "flue.compaction_reason": "manual", -│ "flue.session": "main", -│ "provider": "flue" -│ } -│ metrics: { -│ "duration_ms": 0, -│ "messages_after": 2, -│ "messages_before": 8 -│ } -│ └── flue.turn [llm] -│ input: [ -│ { -│ "content": [ -│ { -│ "text": "\n[User]: Complete this instrumented research flow. Call exactly one tool per turn and wait for each tool result before choosing the next tool. Step 1: call lookup with query \"flue instrumentation\". Step 2: use the lookup result id as lookupId and call web_search with query \"Braintrust Flue reasoning stream instrumentation\". Step 3: use the first web_search result url and call summarize_source. After summarize_source returns, reply with exactly PROMPT_DONE and no other text.\n\n[Assistant thinking]: The user wants me to complete a research flow with exactly three steps:\n1. Call lookup with query \"flue instrumentation\"\n2. Call web_search with the lookupId from step 1 and query \"Braintrust Flue reasoning stream instrumentation\"\n3. Call summarize_source with the first URL from step 2\n4. Reply with exactly \"PROMPT_DONE\"\n\nThe user emphasizes I should call exactly one tool per turn and wait for each result. Let me start with step 1.\n\n[Assistant]: I'll complete this instrumented research flow step by step. Starting with Step 1:\n\n[Assistant tool calls]: lookup(query=\"flue instrumentation\")\n\n[Tool result]: {\"id\":\"flue-session-2026\",\"query\":\"flue instrumentation\",\"topic\":\"session instrumentation\"}\n\n[Assistant thinking]: Good, I got the lookup result with id \"flue-session-2026\". Now I need to proceed to step 2: call web_search with this lookupId and the query \"Braintrust Flue reasoning stream instrumentation\".\n\n[Assistant]: Now proceeding to Step 2:\n\n[Assistant tool calls]: web_search(lookupId=\"flue-session-2026\", query=\"Braintrust Flue reasoning stream instrumentation\")\n\n[Tool result]: {\"lookupId\":\"flue-session-2026\",\"query\":\"Braintrust Flue reasoning stream instrumentation\",\"results\":[{\"title\":\"Flue reasoning stream instrumentation\",\"url\":\"https://example.test/flue/reasoning-streams\"}]}\n\n[Assistant thinking]: Perfect! I got the web_search results with one result. The first (and only) result has the URL \"https://example.test/flue/reasoning-streams\". Now I need to proceed to step 3: call summarize_source with this URL.\n\n[Assistant]: Now proceeding to Step 3:\n\n[Assistant tool calls]: summarize_source(url=\"https://example.test/flue/reasoning-streams\")\n\n[Tool result]: {\"summary\":\"Flue emits reasoning, tool execution, and LLM turn events separately.\",\"url\":\"https://example.test/flue/reasoning-streams\"}\n\n\nThis is the PREFIX of a turn that was too large to keep. The SUFFIX (recent work) is retained.\n\nSummarize the prefix to provide context for the retained suffix:\n\n## Original Request\n[What did the user ask for in this turn?]\n\n## Early Progress\n- [Key decisions and work done in the prefix]\n\n## Context for Suffix\n- [Information needed to understand the retained recent work]\n\nBe concise. Focus on what's needed to understand the kept suffix.", -│ "type": "text" -│ } -│ ], -│ "role": "user" -│ } -│ ] -│ output: { -│ "content": [ -│ { -│ "text": "## Original Request\nThe user requested to complete an instrumented research flow consisting of three specific steps, involving calling various tools in a sequential manner.\n\n## Early Progress", -│ "textSignature": "{\"v\":1,\"id\":\"msg_01ecf68325f9e202016a184601309081a0878ef4054e950d32\"}", -│ "type": "text" -│ } -│ ], -│ "role": "assistant" -│ } -│ metadata: { -│ "flue.api": "openai-responses", -│ "flue.model": "gpt-4o-mini", -│ "flue.provider": "openai", -│ "flue.session": "main", -│ "flue.stop_reason": "stop", -│ "flue.turn_purpose": "compaction_prefix", -│ "model": "gpt-4o-mini", -│ "provider": "openai" -│ } -│ metrics: { -│ "completion_tokens": 0, -│ "duration_ms": 0, -│ "estimated_cost": 0, -│ "prompt_cache_creation_tokens": 0, -│ "prompt_cached_tokens": 0, -│ "prompt_tokens": 0, -│ "tokens": 0 -│ } -└── flue.prompt [task] - input: [ - { - "content": [ - { - "text": "Complete this instrumented research flow. Call exactly one tool per turn and wait for each tool result before choosing the next tool. Step 1: call lookup with query \"flue instrumentation\". Step 2: use the lookup result id as lookupId and call web_search with query \"Braintrust Flue reasoning stream instrumentation\". Step 3: use the first web_search result url and call summarize_source. After summarize_source returns, reply with exactly PROMPT_DONE and no other text.", - "type": "text" - } - ], - "role": "user" - } - ] - output: "PROMPT_DONE" - metadata: { - "flue.operation": "prompt", - "flue.session": "main", - "flue.workflow_name": "instrumentation", - "provider": "flue", - "scenario": "flue-instrumentation" - } - metrics: { - "duration_ms": 0 - } - ├── flue.turn [llm] - │ input: [ - │ { - │ "content": [ - │ { - │ "text": "Complete this instrumented research flow. Call exactly one tool per turn and wait for each tool result before choosing the next tool. Step 1: call lookup with query \"flue instrumentation\". Step 2: use the lookup result id as lookupId and call web_search with query \"Braintrust Flue reasoning stream instrumentation\". Step 3: use the first web_search result url and call summarize_source. After summarize_source returns, reply with exactly PROMPT_DONE and no other text.", - │ "type": "text" - │ } - │ ], - │ "role": "user" - │ } - │ ] - │ output: { - │ "content": [ - │ { - │ "thinking": "The user wants me to complete a research flow with exactly three steps:\n1. Call lookup with query \"flue instrumentation\"\n2. Call web_search with the lookupId from step 1 and query \"Braintrust Flue reasoning stream instrumentation\"\n3. Call summarize_source with the first URL from step 2\n4. Reply with exactly \"PROMPT_DONE\"\n\nThe user emphasizes I should call exactly one tool per turn and wait for each result. Let me start with step 1.", - │ "thinkingSignature": "EoQFCm4IDhgCKkD+AoKMhpXNPUJ5eANwM28uUCJPCE+AXh0ACw7aXniX+YN04zdJl3hlHMbtAuKhhczLiEeOJtu4KtVpea5u5NFdMhpjbGF1ZGUtc29ubmV0LTQtNS0yMDI1MDkyOTgAQgh0aGlua2luZxIMIOTWCNEtsiI9vF4SGgwPXC861uZx8lpOziEiMLsCukGIluD0U44degzYnHFc+BwXNIEB+yatSHlZE+a5smr9crZcft3OLmsIdXuaFSrDAyQxWd9oqfJdnvza9eWIaOCeNkVCjrTc/wotBwh5Xp8m9k/WKt7P4zzUJBuLQ1f6ZnsdWkL8OiL/2E7Up55z2+UuVWsg1Y90/x27YRFdQjv3RnztYp4bZ5oIHAlTJoCzWiBYEh/Z8c2O+wy6MHo1cp0T57OroB/4HAqRZxrjTOXpKYLmYKXAY5tU8DTd3A0/FfEhg3oPSbOhX+112szFCpm3BRNJYooUUhwSu1hO4H/6r7d/Yp9XIM4tBfzuDx83CC/YetdOFgnYkdliw0xaUGaj4o37rWIRctWHAMdlUc4NDA0Ljc1uirHDXkN8EJwWSoIjPSBTA/p/wgLrvRV5Yy9Z0dMoWOpgYF6t1H/tNAyY8ml1CA7IWyd3jnfTZ1/G3Y0xFDwJDFlb4/QKc67w36SSCa+X9mMoTxD/DYN249lsII+XX3lQ7F1zJ07THOtCyRirCaRhmE/kGwl5tiB1QHrWa9bNHxQxf9i92435JGKN053Fe14AR70MtBSzIU2/zbgvhB1vF5lZt2i46hNxfo4lXR495gEwdwC6+8HJnlEJAJkdeHfZKj+RiwXoiYKPKsSbHi83X8Hh32IifuglNxdlxVQYAQ==", - │ "type": "thinking" - │ }, - │ { - │ "text": "I'll complete this instrumented research flow step by step. Starting with Step 1:", - │ "type": "text" - │ }, - │ { - │ "arguments": { - │ "query": "flue instrumentation" - │ }, - │ "id": "", - │ "name": "lookup", - │ "type": "toolCall" - │ } - │ ], - │ "role": "assistant" - │ } - │ metadata: { - │ "flue.api": "anthropic-messages", - │ "flue.input_mode": "full", - │ "flue.model": "claude-sonnet-4-5-20250929", - │ "flue.provider": "anthropic", - │ "flue.session": "main", - │ "flue.stop_reason": "toolUse", - │ "flue.turn_purpose": "agent", - │ "model": "claude-sonnet-4-5-20250929", - │ "provider": "anthropic" - │ } - │ metrics: { - │ "completion_tokens": 196, - │ "duration_ms": 0, - │ "estimated_cost": 0.0113025, - │ "prompt_cache_creation_tokens": 2222, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 10, - │ "tokens": 2428 - │ } - ├── tool:lookup [tool] - │ input: { - │ "query": "flue instrumentation" - │ } - │ output: { - │ "content": [ - │ { - │ "text": "{\"id\":\"flue-session-2026\",\"query\":\"flue instrumentation\",\"topic\":\"session instrumentation\"}", - │ "type": "text" - │ } - │ ], - │ "details": { - │ "customTool": "lookup" - │ } - │ } - │ metadata: { - │ "flue.session": "main", - │ "flue.tool_name": "lookup", - │ "provider": "flue" - │ } - │ metrics: { - │ "duration_ms": 0 - │ } - ├── flue.turn [llm] - │ input: [ - │ { - │ "content": [ - │ { - │ "thinking": "The user wants me to complete a research flow with exactly three steps:\n1. Call lookup with query \"flue instrumentation\"\n2. Call web_search with the lookupId from step 1 and query \"Braintrust Flue reasoning stream instrumentation\"\n3. Call summarize_source with the first URL from step 2\n4. Reply with exactly \"PROMPT_DONE\"\n\nThe user emphasizes I should call exactly one tool per turn and wait for each result. Let me start with step 1.", - │ "thinkingSignature": "EoQFCm4IDhgCKkD+AoKMhpXNPUJ5eANwM28uUCJPCE+AXh0ACw7aXniX+YN04zdJl3hlHMbtAuKhhczLiEeOJtu4KtVpea5u5NFdMhpjbGF1ZGUtc29ubmV0LTQtNS0yMDI1MDkyOTgAQgh0aGlua2luZxIMIOTWCNEtsiI9vF4SGgwPXC861uZx8lpOziEiMLsCukGIluD0U44degzYnHFc+BwXNIEB+yatSHlZE+a5smr9crZcft3OLmsIdXuaFSrDAyQxWd9oqfJdnvza9eWIaOCeNkVCjrTc/wotBwh5Xp8m9k/WKt7P4zzUJBuLQ1f6ZnsdWkL8OiL/2E7Up55z2+UuVWsg1Y90/x27YRFdQjv3RnztYp4bZ5oIHAlTJoCzWiBYEh/Z8c2O+wy6MHo1cp0T57OroB/4HAqRZxrjTOXpKYLmYKXAY5tU8DTd3A0/FfEhg3oPSbOhX+112szFCpm3BRNJYooUUhwSu1hO4H/6r7d/Yp9XIM4tBfzuDx83CC/YetdOFgnYkdliw0xaUGaj4o37rWIRctWHAMdlUc4NDA0Ljc1uirHDXkN8EJwWSoIjPSBTA/p/wgLrvRV5Yy9Z0dMoWOpgYF6t1H/tNAyY8ml1CA7IWyd3jnfTZ1/G3Y0xFDwJDFlb4/QKc67w36SSCa+X9mMoTxD/DYN249lsII+XX3lQ7F1zJ07THOtCyRirCaRhmE/kGwl5tiB1QHrWa9bNHxQxf9i92435JGKN053Fe14AR70MtBSzIU2/zbgvhB1vF5lZt2i46hNxfo4lXR495gEwdwC6+8HJnlEJAJkdeHfZKj+RiwXoiYKPKsSbHi83X8Hh32IifuglNxdlxVQYAQ==", - │ "type": "thinking" - │ }, - │ { - │ "text": "I'll complete this instrumented research flow step by step. Starting with Step 1:", - │ "type": "text" - │ }, - │ { - │ "arguments": { - │ "query": "flue instrumentation" - │ }, - │ "id": "", - │ "name": "lookup", - │ "type": "toolCall" - │ } - │ ], - │ "role": "assistant" - │ }, - │ { - │ "content": [ - │ { - │ "text": "{\"id\":\"flue-session-2026\",\"query\":\"flue instrumentation\",\"topic\":\"session instrumentation\"}", - │ "type": "text" - │ } - │ ], - │ "isError": false, - │ "role": "toolResult", - │ "toolCallId": "", - │ "toolName": "lookup" - │ } - │ ] - │ output: { - │ "content": [ - │ { - │ "thinking": "Good, I got the lookup result with id \"flue-session-2026\". Now I need to proceed to step 2: call web_search with this lookupId and the query \"Braintrust Flue reasoning stream instrumentation\".", - │ "thinkingSignature": "EpEDCm4IDhgCKkAMahve1kB1j4njp1ZSzO8eCK4bb5Ex7MB6tjdoOXfuUDRfkBqvIGdwVly8GiA8GGYC7Y/RVfREXF6A6OXHTwFyMhpjbGF1ZGUtc29ubmV0LTQtNS0yMDI1MDkyOTgAQgh0aGlua2luZxIMLw+fsa4yo7a7ZSoIGgyaTdvH020n2dQj9bUiMB4OokY29w+uZb4rqDnKDbLXUqUj6U0f1QCGQF2ZBYDDfDJyM5Jv1xY3rQiUUjkhBirQAS4XivjvMnWI5zHBmpI/u/pq4aGdrdb3vUsJ/ne6R1d80JA/INGF8vLkQcZ5xxEK0jU4yRE+ldc2uzavmtoxIKAFD6bqKks1Qv6u+TIZtuW7tUB/7jFxDw93KmJNruMIa8c1J6pPUyBy3xE7QwgmU0kDfIfdnljxLoEBiMwtnxjCPj9lT7HHz4FkrhbnQTV4MvAQohCLjaSbOTd6LdLxwtJPm9mc26ID4eYN1gCCzyAxGXhqLHlzc23PafvQjsZliWnLhaiNMnG0YEo78XkuJQsYAQ==", - │ "type": "thinking" - │ }, - │ { - │ "text": "Now proceeding to Step 2:", - │ "type": "text" - │ }, - │ { - │ "arguments": { - │ "lookupId": "flue-session-2026", - │ "query": "Braintrust Flue reasoning stream instrumentation" - │ }, - │ "id": "", - │ "name": "web_search", - │ "type": "toolCall" - │ } - │ ], - │ "role": "assistant" - │ } - │ metadata: { - │ "flue.api": "anthropic-messages", - │ "flue.input_message_offset": 1, - │ "flue.input_mode": "delta", - │ "flue.model": "claude-sonnet-4-5-20250929", - │ "flue.provider": "anthropic", - │ "flue.session": "main", - │ "flue.stop_reason": "toolUse", - │ "flue.turn_purpose": "agent", - │ "model": "claude-sonnet-4-5-20250929", - │ "provider": "anthropic" - │ } - │ metrics: { - │ "completion_tokens": 158, - │ "duration_ms": 0, - │ "estimated_cost": 0.0039606, - │ "prompt_cache_creation_tokens": 236, - │ "prompt_cached_tokens": 2222, - │ "prompt_tokens": 13, - │ "tokens": 2629 - │ } - ├── tool:web_search [tool] - │ input: { - │ "lookupId": "flue-session-2026", - │ "query": "Braintrust Flue reasoning stream instrumentation" - │ } - │ output: { - │ "content": [ - │ { - │ "text": "{\"lookupId\":\"flue-session-2026\",\"query\":\"Braintrust Flue reasoning stream instrumentation\",\"results\":[{\"title\":\"Flue reasoning stream instrumentation\",\"url\":\"https://example.test/flue/reasoning-streams\"}]}", - │ "type": "text" - │ } - │ ], - │ "details": { - │ "customTool": "web_search" - │ } - │ } - │ metadata: { - │ "flue.session": "main", - │ "flue.tool_name": "web_search", - │ "provider": "flue" - │ } - │ metrics: { - │ "duration_ms": 0 - │ } - ├── flue.turn [llm] - │ input: [ - │ { - │ "content": [ - │ { - │ "thinking": "Good, I got the lookup result with id \"flue-session-2026\". Now I need to proceed to step 2: call web_search with this lookupId and the query \"Braintrust Flue reasoning stream instrumentation\".", - │ "thinkingSignature": "EpEDCm4IDhgCKkAMahve1kB1j4njp1ZSzO8eCK4bb5Ex7MB6tjdoOXfuUDRfkBqvIGdwVly8GiA8GGYC7Y/RVfREXF6A6OXHTwFyMhpjbGF1ZGUtc29ubmV0LTQtNS0yMDI1MDkyOTgAQgh0aGlua2luZxIMLw+fsa4yo7a7ZSoIGgyaTdvH020n2dQj9bUiMB4OokY29w+uZb4rqDnKDbLXUqUj6U0f1QCGQF2ZBYDDfDJyM5Jv1xY3rQiUUjkhBirQAS4XivjvMnWI5zHBmpI/u/pq4aGdrdb3vUsJ/ne6R1d80JA/INGF8vLkQcZ5xxEK0jU4yRE+ldc2uzavmtoxIKAFD6bqKks1Qv6u+TIZtuW7tUB/7jFxDw93KmJNruMIa8c1J6pPUyBy3xE7QwgmU0kDfIfdnljxLoEBiMwtnxjCPj9lT7HHz4FkrhbnQTV4MvAQohCLjaSbOTd6LdLxwtJPm9mc26ID4eYN1gCCzyAxGXhqLHlzc23PafvQjsZliWnLhaiNMnG0YEo78XkuJQsYAQ==", - │ "type": "thinking" - │ }, - │ { - │ "text": "Now proceeding to Step 2:", - │ "type": "text" - │ }, - │ { - │ "arguments": { - │ "lookupId": "flue-session-2026", - │ "query": "Braintrust Flue reasoning stream instrumentation" - │ }, - │ "id": "", - │ "name": "web_search", - │ "type": "toolCall" - │ } - │ ], - │ "role": "assistant" - │ }, - │ { - │ "content": [ - │ { - │ "text": "{\"lookupId\":\"flue-session-2026\",\"query\":\"Braintrust Flue reasoning stream instrumentation\",\"results\":[{\"title\":\"Flue reasoning stream instrumentation\",\"url\":\"https://example.test/flue/reasoning-streams\"}]}", - │ "type": "text" - │ } - │ ], - │ "isError": false, - │ "role": "toolResult", - │ "toolCallId": "", - │ "toolName": "web_search" - │ } - │ ] - │ output: { - │ "content": [ - │ { - │ "thinking": "Perfect! I got the web_search results with one result. The first (and only) result has the URL \"https://example.test/flue/reasoning-streams\". Now I need to proceed to step 3: call summarize_source with this URL.", - │ "thinkingSignature": "EqQDCm4IDhgCKkAXv7EDAhWMcrBz11bUd5qxlzyy03CAGQfRgLHDE0uKzLATh1o2sedd8GxLwwQqV0CmLRfNxsdK/ZDp3Lks9iysMhpjbGF1ZGUtc29ubmV0LTQtNS0yMDI1MDkyOTgAQgh0aGlua2luZxIMXDMniOy5Dy0KI8x7GgzgHs7BYJMs6/N0ur8iMPe0SwnHhP39OjEu7P8ASwK9ffwa3VP67QOlPzOstRRdEZNd0mq+BVjpmBXB+QcAzCrjAeAh/UO/d7SfFmXjqwA3//thoL/TTlGa6W+fdOCFRikn65JWT4ZdVFetfqD+9/QCP/nXEwMsLnTBAxC4EGryrgs2YT+OWwmEJ7rHKFeDs5djhwSb2J2ZJO1kRvD9s2vCCDkImygYBSIVgx1Y0CZJHhzn2Vf39v0Fbth2wDZa+OqxivV9NmDZEjmgcM+tTRx1CCnaww8LZEJQ68v/dUJy7hhD4MrNcQ62evJYPfT3+4EKKWJ/7HI7KXz91//+YAkph63XstbJMFJ3AldAFiq3ecauXt5r42N5gmCmwZX+sI48ENLGGAE=", - │ "type": "thinking" - │ }, - │ { - │ "text": "Now proceeding to Step 3:", - │ "type": "text" - │ }, - │ { - │ "arguments": { - │ "url": "https://example.test/flue/reasoning-streams" - │ }, - │ "id": "", - │ "name": "summarize_source", - │ "type": "toolCall" - │ } - │ ], - │ "role": "assistant" - │ } - │ metadata: { - │ "flue.api": "anthropic-messages", - │ "flue.input_message_offset": 3, - │ "flue.input_mode": "delta", - │ "flue.model": "claude-sonnet-4-5-20250929", - │ "flue.provider": "anthropic", - │ "flue.session": "main", - │ "flue.stop_reason": "toolUse", - │ "flue.turn_purpose": "agent", - │ "model": "claude-sonnet-4-5-20250929", - │ "provider": "anthropic" - │ } - │ metrics: { - │ "completion_tokens": 138, - │ "duration_ms": 0, - │ "estimated_cost": 0.0037239000000000005, - │ "prompt_cache_creation_tokens": 234, - │ "prompt_cached_tokens": 2458, - │ "prompt_tokens": 13, - │ "tokens": 2843 - │ } - ├── tool:summarize_source [tool] - │ input: { - │ "url": "https://example.test/flue/reasoning-streams" - │ } - │ output: { - │ "content": [ - │ { - │ "text": "{\"summary\":\"Flue emits reasoning, tool execution, and LLM turn events separately.\",\"url\":\"https://example.test/flue/reasoning-streams\"}", - │ "type": "text" - │ } - │ ], - │ "details": { - │ "customTool": "summarize_source" - │ } - │ } - │ metadata: { - │ "flue.session": "main", - │ "flue.tool_name": "summarize_source", - │ "provider": "flue" - │ } - │ metrics: { - │ "duration_ms": 0 - │ } - └── flue.turn [llm] - input: [ - { - "content": [ - { - "thinking": "Perfect! I got the web_search results with one result. The first (and only) result has the URL \"https://example.test/flue/reasoning-streams\". Now I need to proceed to step 3: call summarize_source with this URL.", - "thinkingSignature": "EqQDCm4IDhgCKkAXv7EDAhWMcrBz11bUd5qxlzyy03CAGQfRgLHDE0uKzLATh1o2sedd8GxLwwQqV0CmLRfNxsdK/ZDp3Lks9iysMhpjbGF1ZGUtc29ubmV0LTQtNS0yMDI1MDkyOTgAQgh0aGlua2luZxIMXDMniOy5Dy0KI8x7GgzgHs7BYJMs6/N0ur8iMPe0SwnHhP39OjEu7P8ASwK9ffwa3VP67QOlPzOstRRdEZNd0mq+BVjpmBXB+QcAzCrjAeAh/UO/d7SfFmXjqwA3//thoL/TTlGa6W+fdOCFRikn65JWT4ZdVFetfqD+9/QCP/nXEwMsLnTBAxC4EGryrgs2YT+OWwmEJ7rHKFeDs5djhwSb2J2ZJO1kRvD9s2vCCDkImygYBSIVgx1Y0CZJHhzn2Vf39v0Fbth2wDZa+OqxivV9NmDZEjmgcM+tTRx1CCnaww8LZEJQ68v/dUJy7hhD4MrNcQ62evJYPfT3+4EKKWJ/7HI7KXz91//+YAkph63XstbJMFJ3AldAFiq3ecauXt5r42N5gmCmwZX+sI48ENLGGAE=", - "type": "thinking" - }, - { - "text": "Now proceeding to Step 3:", - "type": "text" - }, - { - "arguments": { - "url": "https://example.test/flue/reasoning-streams" - }, - "id": "", - "name": "summarize_source", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "{\"summary\":\"Flue emits reasoning, tool execution, and LLM turn events separately.\",\"url\":\"https://example.test/flue/reasoning-streams\"}", - "type": "text" - } - ], - "isError": false, - "role": "toolResult", - "toolCallId": "", - "toolName": "summarize_source" - } - ] - output: { - "content": [ - { - "thinking": "Perfect! The summarize_source call has completed successfully. According to the instructions, I should now reply with exactly \"PROMPT_DONE\" and no other text.", - "thinkingSignature": "Eu8CCm4IDhgCKkAn2QOIIlYfvcs6ynsmhCslHDtBsrTA4CfmimK7zyz6+k6dZldV2JBtEBET41It0gFcGEKC0n0mAhaYVuLutpDDMhpjbGF1ZGUtc29ubmV0LTQtNS0yMDI1MDkyOTgAQgh0aGlua2luZxIM1xfLptxSF8rKzou8GgwRLuVmfiBYOT4CZIsiMAYz8TpvchMAWodP+T9uZ+dJ47MVdNv+p86nZXmT5d5EWlCroiAOFHHzJH2o0UjJQSquAX6wu+jIC8/DXwurGIIf2ToldPuJWF5HGj8kHMnwh777XS1NUFl1irM24rvP7kTAv3Dp8YEwb8t3g1EsicY7j+C5DzgwxVsPYCVOCgdMqQjtsqpL7AX5KQ9+hR1gK1DQjshv4Gze3hw/+C+9iptd3RlDIuwO0cm8NY+sVqeWVh3TtQujQlVt1rW/HfWjclrtT0LZSGwc6RypmfMMQi6lncIk/EPgy0XbS1plKW0tLBgB", - "type": "thinking" - }, - { - "text": "PROMPT_DONE", - "type": "text" - } - ], - "role": "assistant" - } - metadata: { - "flue.api": "anthropic-messages", - "flue.input_message_offset": 5, - "flue.input_mode": "delta", - "flue.model": "claude-sonnet-4-5-20250929", - "flue.provider": "anthropic", - "flue.session": "main", - "flue.stop_reason": "stop", - "flue.turn_purpose": "agent", - "model": "claude-sonnet-4-5-20250929", - "provider": "anthropic" - } - metrics: { - "completion_tokens": 51, - "duration_ms": 0, - "estimated_cost": 0.0023391, - "prompt_cache_creation_tokens": 194, - "prompt_cached_tokens": 2692, - "prompt_tokens": 13, - "tokens": 2950 - } diff --git a/e2e/scenarios/flue-instrumentation/__snapshots__/flue-v0-8-latest-auto-hook.span-tree.json b/e2e/scenarios/flue-instrumentation/__snapshots__/flue-v0-8-latest-auto-hook.span-tree.json deleted file mode 100644 index e26112b02..000000000 --- a/e2e/scenarios/flue-instrumentation/__snapshots__/flue-v0-8-latest-auto-hook.span-tree.json +++ /dev/null @@ -1,1662 +0,0 @@ -{ - "span_tree": [ - { - "name": "workflow:instrumentation", - "type": "task", - "children": [ - { - "name": "flue.skill", - "type": "task", - "children": [ - { - "name": "flue.turn", - "type": "llm", - "children": [ - { - "name": "flue.task", - "type": "task", - "children": [ - { - "name": "flue.turn", - "type": "llm", - "children": [], - "input": [ - { - "content": [ - { - "text": "Run the skill named \"e2e-flue-skill\" with arguments {\"marker\":\"SKILL_DONE\"}. Output only the marker and no extra text when finished.", - "type": "text" - } - ], - "role": "user" - } - ], - "output": { - "content": [ - { - "arguments": { - "agent": "e2e-flue-skill", - "cwd": ".", - "description": "Run e2e-flue-skill with marker SKILL_DONE", - "prompt": "Run the skill named \"e2e-flue-skill\" with arguments {\"marker\":\"SKILL_DONE\"}. Ensure completion and report the final output marker only." - }, - "id": "", - "name": "task", - "type": "toolCall" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "openai-responses", - "flue.input_mode": "full", - "flue.model": "gpt-5.4-nano", - "flue.provider": "openai", - "flue.session": "task:skill:", - "flue.stop_reason": "toolUse", - "flue.turn_purpose": "agent", - "model": "gpt-5.4-nano", - "provider": "openai" - }, - "metrics": { - "completion_tokens": 113, - "duration_ms": 0, - "estimated_cost": 0.00031225000000000003, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 855, - "tokens": 968 - } - }, - { - "name": "tool:task", - "type": "tool", - "children": [], - "input": { - "agent": "e2e-flue-skill", - "cwd": ".", - "description": "Run e2e-flue-skill with marker SKILL_DONE", - "prompt": "Run the skill named \"e2e-flue-skill\" with arguments {\"marker\":\"SKILL_DONE\"}. Ensure completion and report the final output marker only." - }, - "output": { - "content": [ - { - "text": "[flue] Subagent \"e2e-flue-skill\" is not declared. Available: (none).", - "type": "text" - } - ], - "details": {} - }, - "metadata": { - "flue.session": "task:skill:", - "flue.tool_name": "task", - "provider": "flue" - }, - "metrics": { - "duration_ms": 0 - }, - "error": "{\"content\":[{\"type\":\"text\",\"text\":\"[flue] Subagent \\\"e2e-flue-skill\\\" is not declared. Available: (none).\"}],\"details\":{}}" - }, - { - "name": "flue.turn", - "type": "llm", - "children": [], - "input": [ - { - "content": [ - { - "arguments": { - "agent": "e2e-flue-skill", - "cwd": ".", - "description": "Run e2e-flue-skill with marker SKILL_DONE", - "prompt": "Run the skill named \"e2e-flue-skill\" with arguments {\"marker\":\"SKILL_DONE\"}. Ensure completion and report the final output marker only." - }, - "id": "", - "name": "task", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "[flue] Subagent \"e2e-flue-skill\" is not declared. Available: (none).", - "type": "text" - } - ], - "isError": true, - "role": "toolResult", - "toolCallId": "", - "toolName": "task" - } - ], - "output": { - "content": [ - { - "arguments": { - "command": "find . -maxdepth 4 -name AGENTS.md -print", - "timeout": 100000 - }, - "id": "", - "name": "bash", - "type": "toolCall" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "openai-responses", - "flue.input_message_offset": 1, - "flue.input_mode": "delta", - "flue.model": "gpt-5.4-nano", - "flue.provider": "openai", - "flue.session": "task:skill:", - "flue.stop_reason": "toolUse", - "flue.turn_purpose": "agent", - "model": "gpt-5.4-nano", - "provider": "openai" - }, - "metrics": { - "completion_tokens": 94, - "duration_ms": 0, - "estimated_cost": 0.00031790000000000003, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 1002, - "tokens": 1096 - } - }, - { - "name": "tool:bash", - "type": "tool", - "children": [], - "input": { - "command": "find . -maxdepth 4 -name AGENTS.md -print", - "timeout": 100000 - }, - "output": { - "content": [ - { - "text": "(no output)", - "type": "text" - } - ], - "details": { - "command": "find . -maxdepth 4 -name AGENTS.md -print", - "exitCode": 0 - } - }, - "metadata": { - "flue.session": "task:skill:", - "flue.tool_name": "bash", - "provider": "flue" - }, - "metrics": { - "duration_ms": 0 - } - }, - { - "name": "flue.turn", - "type": "llm", - "children": [], - "input": [ - { - "content": [ - { - "arguments": { - "command": "find . -maxdepth 4 -name AGENTS.md -print", - "timeout": 100000 - }, - "id": "", - "name": "bash", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "(no output)", - "type": "text" - } - ], - "isError": false, - "role": "toolResult", - "toolCallId": "", - "toolName": "bash" - } - ], - "output": { - "content": [ - { - "arguments": { - "command": "find . -maxdepth 4 -type f -name AGENTS.md -print -exec sed -n '1,200p' {} \\;", - "timeout": 100000 - }, - "id": "", - "name": "bash", - "type": "toolCall" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "openai-responses", - "flue.input_message_offset": 3, - "flue.input_mode": "delta", - "flue.model": "gpt-5.4-nano", - "flue.provider": "openai", - "flue.session": "task:skill:", - "flue.stop_reason": "toolUse", - "flue.turn_purpose": "agent", - "model": "gpt-5.4-nano", - "provider": "openai" - }, - "metrics": { - "completion_tokens": 87, - "duration_ms": 0, - "estimated_cost": 0.00034815000000000003, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 1197, - "tokens": 1284 - } - }, - { - "name": "tool:bash", - "type": "tool", - "children": [], - "input": { - "command": "find . -maxdepth 4 -type f -name AGENTS.md -print -exec sed -n '1,200p' {} \\;", - "timeout": 100000 - }, - "output": { - "content": [ - { - "text": "(no output)", - "type": "text" - } - ], - "details": { - "command": "find . -maxdepth 4 -type f -name AGENTS.md -print -exec sed -n '1,200p' {} \\;", - "exitCode": 0 - } - }, - "metadata": { - "flue.session": "task:skill:", - "flue.tool_name": "bash", - "provider": "flue" - }, - "metrics": { - "duration_ms": 0 - } - }, - { - "name": "flue.turn", - "type": "llm", - "children": [], - "input": [ - { - "content": [ - { - "arguments": { - "command": "find . -maxdepth 4 -type f -name AGENTS.md -print -exec sed -n '1,200p' {} \\;", - "timeout": 100000 - }, - "id": "", - "name": "bash", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "(no output)", - "type": "text" - } - ], - "isError": false, - "role": "toolResult", - "toolCallId": "", - "toolName": "bash" - } - ], - "output": { - "content": [ - { - "arguments": { - "command": "find . -maxdepth 6 -type f -name 'AGENTS.md' -print", - "timeout": 100000 - }, - "id": "", - "name": "bash", - "type": "toolCall" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "openai-responses", - "flue.input_message_offset": 5, - "flue.input_mode": "delta", - "flue.model": "gpt-5.4-nano", - "flue.provider": "openai", - "flue.session": "task:skill:", - "flue.stop_reason": "toolUse", - "flue.turn_purpose": "agent", - "model": "gpt-5.4-nano", - "provider": "openai" - }, - "metrics": { - "completion_tokens": 58, - "duration_ms": 0, - "estimated_cost": 0.0003321, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 1298, - "tokens": 1356 - } - }, - { - "name": "tool:bash", - "type": "tool", - "children": [], - "input": { - "command": "find . -maxdepth 6 -type f -name 'AGENTS.md' -print", - "timeout": 100000 - }, - "output": { - "content": [ - { - "text": "(no output)", - "type": "text" - } - ], - "details": { - "command": "find . -maxdepth 6 -type f -name 'AGENTS.md' -print", - "exitCode": 0 - } - }, - "metadata": { - "flue.session": "task:skill:", - "flue.tool_name": "bash", - "provider": "flue" - }, - "metrics": { - "duration_ms": 0 - } - }, - { - "name": "flue.turn", - "type": "llm", - "children": [], - "input": [ - { - "content": [ - { - "arguments": { - "command": "find . -maxdepth 6 -type f -name 'AGENTS.md' -print", - "timeout": 100000 - }, - "id": "", - "name": "bash", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "(no output)", - "type": "text" - } - ], - "isError": false, - "role": "toolResult", - "toolCallId": "", - "toolName": "bash" - } - ], - "output": { - "content": [ - { - "arguments": { - "command": "find . -maxdepth 5 -type f -name '*flue*skill*' -o -name '*skill*' | head", - "timeout": 100000 - }, - "id": "", - "name": "bash", - "type": "toolCall" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "openai-responses", - "flue.input_message_offset": 7, - "flue.input_mode": "delta", - "flue.model": "gpt-5.4-nano", - "flue.provider": "openai", - "flue.session": "task:skill:", - "flue.stop_reason": "toolUse", - "flue.turn_purpose": "agent", - "model": "gpt-5.4-nano", - "provider": "openai" - }, - "metrics": { - "completion_tokens": 68, - "duration_ms": 0, - "estimated_cost": 0.00035900000000000005, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 1370, - "tokens": 1438 - } - }, - { - "name": "tool:bash", - "type": "tool", - "children": [], - "input": { - "command": "find . -maxdepth 5 -type f -name '*flue*skill*' -o -name '*skill*' | head", - "timeout": 100000 - }, - "output": { - "content": [ - { - "text": "./.agents/skills\n./.agents/skills/e2e-flue-skill", - "type": "text" - } - ], - "details": { - "command": "find . -maxdepth 5 -type f -name '*flue*skill*' -o -name '*skill*' | head", - "exitCode": 0 - } - }, - "metadata": { - "flue.session": "task:skill:", - "flue.tool_name": "bash", - "provider": "flue" - }, - "metrics": { - "duration_ms": 0 - } - }, - { - "name": "flue.turn", - "type": "llm", - "children": [], - "input": [ - { - "content": [ - { - "arguments": { - "command": "find . -maxdepth 5 -type f -name '*flue*skill*' -o -name '*skill*' | head", - "timeout": 100000 - }, - "id": "", - "name": "bash", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "./.agents/skills\n./.agents/skills/e2e-flue-skill", - "type": "text" - } - ], - "isError": false, - "role": "toolResult", - "toolCallId": "", - "toolName": "bash" - } - ], - "output": { - "content": [ - { - "arguments": { - "command": "find ./.agents/skills/e2e-flue-skill -maxdepth 2 -type f -print", - "timeout": 100000 - }, - "id": "", - "name": "bash", - "type": "toolCall" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "openai-responses", - "flue.input_message_offset": 9, - "flue.input_mode": "delta", - "flue.model": "gpt-5.4-nano", - "flue.provider": "openai", - "flue.session": "task:skill:", - "flue.stop_reason": "toolUse", - "flue.turn_purpose": "agent", - "model": "gpt-5.4-nano", - "provider": "openai" - }, - "metrics": { - "completion_tokens": 109, - "duration_ms": 0, - "estimated_cost": 0.00043885000000000007, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 1513, - "tokens": 1622 - } - }, - { - "name": "tool:bash", - "type": "tool", - "children": [], - "input": { - "command": "find ./.agents/skills/e2e-flue-skill -maxdepth 2 -type f -print", - "timeout": 100000 - }, - "output": { - "content": [ - { - "text": "./.agents/skills/e2e-flue-skill/SKILL.md", - "type": "text" - } - ], - "details": { - "command": "find ./.agents/skills/e2e-flue-skill -maxdepth 2 -type f -print", - "exitCode": 0 - } - }, - "metadata": { - "flue.session": "task:skill:", - "flue.tool_name": "bash", - "provider": "flue" - }, - "metrics": { - "duration_ms": 0 - } - }, - { - "name": "flue.turn", - "type": "llm", - "children": [], - "input": [ - { - "content": [ - { - "arguments": { - "command": "find ./.agents/skills/e2e-flue-skill -maxdepth 2 -type f -print", - "timeout": 100000 - }, - "id": "", - "name": "bash", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "./.agents/skills/e2e-flue-skill/SKILL.md", - "type": "text" - } - ], - "isError": false, - "role": "toolResult", - "toolCallId": "", - "toolName": "bash" - } - ], - "output": { - "content": [ - { - "arguments": { - "limit": 200, - "offset": 1, - "path": "./.agents/skills/e2e-flue-skill/SKILL.md" - }, - "id": "", - "name": "read", - "type": "toolCall" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "openai-responses", - "flue.input_message_offset": 11, - "flue.input_mode": "delta", - "flue.model": "gpt-5.4-nano", - "flue.provider": "openai", - "flue.session": "task:skill:", - "flue.stop_reason": "toolUse", - "flue.turn_purpose": "agent", - "model": "gpt-5.4-nano", - "provider": "openai" - }, - "metrics": { - "completion_tokens": 64, - "duration_ms": 0, - "estimated_cost": 0.00015592, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 1536, - "prompt_tokens": 226, - "tokens": 1826 - } - }, - { - "name": "tool:read", - "type": "tool", - "children": [], - "input": { - "limit": 200, - "offset": 1, - "path": "./.agents/skills/e2e-flue-skill/SKILL.md" - }, - "output": { - "content": [ - { - "text": "---\nname: e2e-flue-skill\ndescription: Deterministic skill used by the Braintrust Flue instrumentation e2e test.\n---\n\nReturn the marker from the provided args exactly once. Output no other text.\n", - "type": "text" - } - ], - "details": { - "lines": 7, - "path": "./.agents/skills/e2e-flue-skill/SKILL.md" - } - }, - "metadata": { - "flue.session": "task:skill:", - "flue.tool_name": "read", - "provider": "flue" - }, - "metrics": { - "duration_ms": 0 - } - }, - { - "name": "flue.turn", - "type": "llm", - "children": [], - "input": [ - { - "content": [ - { - "arguments": { - "limit": 200, - "offset": 1, - "path": "./.agents/skills/e2e-flue-skill/SKILL.md" - }, - "id": "", - "name": "read", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "---\nname: e2e-flue-skill\ndescription: Deterministic skill used by the Braintrust Flue instrumentation e2e test.\n---\n\nReturn the marker from the provided args exactly once. Output no other text.\n", - "type": "text" - } - ], - "isError": false, - "role": "toolResult", - "toolCallId": "", - "toolName": "read" - } - ], - "output": { - "content": [ - { - "arguments": { - "command": "cat ./package.json", - "timeout": 100000 - }, - "id": "", - "name": "bash", - "type": "toolCall" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "openai-responses", - "flue.input_message_offset": 13, - "flue.input_mode": "delta", - "flue.model": "gpt-5.4-nano", - "flue.provider": "openai", - "flue.session": "task:skill:", - "flue.stop_reason": "toolUse", - "flue.turn_purpose": "agent", - "model": "gpt-5.4-nano", - "provider": "openai" - }, - "metrics": { - "completion_tokens": 90, - "duration_ms": 0, - "estimated_cost": 0.00021242000000000003, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 1536, - "prompt_tokens": 346, - "tokens": 1972 - } - }, - { - "name": "tool:bash", - "type": "tool", - "children": [], - "input": { - "command": "cat ./package.json", - "timeout": 100000 - }, - "output": { - "content": [ - { - "text": "{\n \"name\": \"flue-instrumentation-scenario\",\n \"private\": true,\n \"type\": \"module\",\n \"braintrustScenario\": {\n \"bump\": {\n \"dependencies\": {\n \"flue-cli-v0-8-latest\": {\n \"package\": \"@flue/cli\",\n \"range\": \">=0.8.0 <0.9.0\"\n },\n \"flue-cli-v1-latest\": {\n \"package\": \"@flue/cli\",\n \"range\": \">=1.0.0-beta.3 <2.0.0-0\",\n \"allowPrerelease\": true\n },\n \"flue-runtime-v0-8-latest\": {\n \"package\": \"@flue/runtime\",\n \"range\": \">=0.8.0 <0.9.0\"\n },\n \"flue-runtime-v1-latest\": {\n \"package\": \"@flue/runtime\",\n \"range\": \">=1.0.0-beta.3 <2.0.0-0\",\n \"allowPrerelease\": true\n },\n \"flue-runtime-v2-latest\": {\n \"package\": \"@flue/runtime\",\n \"range\": \">=2.0.0 <3.0.0\"\n }\n }\n }\n },\n \"dependencies\": {\n \"@flue/cli\": \"0.8.0\",\n \"@flue/runtime\": \"0.8.0\",\n \"flue-cli-v0-8-latest\": \"npm:@flue/cli@0.8.1\",\n \"flue-cli-v1\": \"npm:@flue/cli@1.0.0-beta.3\",\n \"flue-cli-v1-latest\": \"npm:@flue/cli@1.0.0-beta.9\",\n \"flue-runtime-v0-8-latest\": \"npm:@flue/runtime@0.8.1\",\n \"flue-runtime-v1\": \"npm:@flue/runtime@1.0.0-beta.3\",\n \"flue-runtime-v1-latest\": \"npm:@flue/runtime@1.0.0-beta.9\",\n \"flue-runtime-v2\": \"npm:@flue/runtime@2.0.0\",\n \"flue-runtime-v2-latest\": \"npm:@flue/runtime@2.0.1\",\n \"pi-ai-v2\": \"npm:@earendil-works/pi-ai@0.83.0\",\n \"valibot\": \"1.4.2\"\n },\n \"pnpm\": {\n \"overrides\": {\n \"@flue/runtime@0.8.0>@earendil-works/pi-agent-core\": \"0.75.4\",\n \"@flue/runtime@0.8.0>@earendil-works/pi-ai\": \"0.75.4\",\n \"@flue/runtime@0.8.1>@earendil-works/pi-agent-core\": \"0.75.4\",\n \"@flue/runtime@0.8.1>@earendil-works/pi-ai\": \"0.75.4\",\n \"@hono/node-server@>=2.0.0 <2.0.10\": \"^2.0.10\",\n \"fast-uri@<3.1.5\": \"^3.1.5\",\n \"hono@>=4.0.0 <4.12.27\": \"^4.12.27\",\n \"brace-expansion@>=3.0.0 <5.0.9\": \"^5.0.9\",\n \"postcss@<8.5.18\": \"^8.5.18\",\n \"valibot@<1.4.2\": \"^1.4.2\",\n \"ip-address@<10.2.2\": \"^10.2.2\",\n \"undici@<7.29.0\": \"^7.29.0\"\n }\n }\n}", - "type": "text" - } - ], - "details": { - "command": "cat ./package.json", - "exitCode": 0 - } - }, - "metadata": { - "flue.session": "task:skill:", - "flue.tool_name": "bash", - "provider": "flue" - }, - "metrics": { - "duration_ms": 0 - } - }, - { - "name": "flue.turn", - "type": "llm", - "children": [], - "input": [ - { - "content": [ - { - "arguments": { - "command": "cat ./package.json", - "timeout": 100000 - }, - "id": "", - "name": "bash", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "{\n \"name\": \"flue-instrumentation-scenario\",\n \"private\": true,\n \"type\": \"module\",\n \"braintrustScenario\": {\n \"bump\": {\n \"dependencies\": {\n \"flue-cli-v0-8-latest\": {\n \"package\": \"@flue/cli\",\n \"range\": \">=0.8.0 <0.9.0\"\n },\n \"flue-cli-v1-latest\": {\n \"package\": \"@flue/cli\",\n \"range\": \">=1.0.0-beta.3 <2.0.0-0\",\n \"allowPrerelease\": true\n },\n \"flue-runtime-v0-8-latest\": {\n \"package\": \"@flue/runtime\",\n \"range\": \">=0.8.0 <0.9.0\"\n },\n \"flue-runtime-v1-latest\": {\n \"package\": \"@flue/runtime\",\n \"range\": \">=1.0.0-beta.3 <2.0.0-0\",\n \"allowPrerelease\": true\n },\n \"flue-runtime-v2-latest\": {\n \"package\": \"@flue/runtime\",\n \"range\": \">=2.0.0 <3.0.0\"\n }\n }\n }\n },\n \"dependencies\": {\n \"@flue/cli\": \"0.8.0\",\n \"@flue/runtime\": \"0.8.0\",\n \"flue-cli-v0-8-latest\": \"npm:@flue/cli@0.8.1\",\n \"flue-cli-v1\": \"npm:@flue/cli@1.0.0-beta.3\",\n \"flue-cli-v1-latest\": \"npm:@flue/cli@1.0.0-beta.9\",\n \"flue-runtime-v0-8-latest\": \"npm:@flue/runtime@0.8.1\",\n \"flue-runtime-v1\": \"npm:@flue/runtime@1.0.0-beta.3\",\n \"flue-runtime-v1-latest\": \"npm:@flue/runtime@1.0.0-beta.9\",\n \"flue-runtime-v2\": \"npm:@flue/runtime@2.0.0\",\n \"flue-runtime-v2-latest\": \"npm:@flue/runtime@2.0.1\",\n \"pi-ai-v2\": \"npm:@earendil-works/pi-ai@0.83.0\",\n \"valibot\": \"1.4.2\"\n },\n \"pnpm\": {\n \"overrides\": {\n \"@flue/runtime@0.8.0>@earendil-works/pi-agent-core\": \"0.75.4\",\n \"@flue/runtime@0.8.0>@earendil-works/pi-ai\": \"0.75.4\",\n \"@flue/runtime@0.8.1>@earendil-works/pi-agent-core\": \"0.75.4\",\n \"@flue/runtime@0.8.1>@earendil-works/pi-ai\": \"0.75.4\",\n \"@hono/node-server@>=2.0.0 <2.0.10\": \"^2.0.10\",\n \"fast-uri@<3.1.5\": \"^3.1.5\",\n \"hono@>=4.0.0 <4.12.27\": \"^4.12.27\",\n \"brace-expansion@>=3.0.0 <5.0.9\": \"^5.0.9\",\n \"postcss@<8.5.18\": \"^8.5.18\",\n \"valibot@<1.4.2\": \"^1.4.2\",\n \"ip-address@<10.2.2\": \"^10.2.2\",\n \"undici@<7.29.0\": \"^7.29.0\"\n }\n }\n}", - "type": "text" - } - ], - "isError": false, - "role": "toolResult", - "toolCallId": "", - "toolName": "bash" - } - ], - "output": { - "content": [ - { - "arguments": { - "command": "printf SKILL_DONE", - "timeout": 100000 - }, - "id": "", - "name": "bash", - "type": "toolCall" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "openai-responses", - "flue.input_message_offset": 15, - "flue.input_mode": "delta", - "flue.model": "gpt-5.4-nano", - "flue.provider": "openai", - "flue.session": "task:skill:", - "flue.stop_reason": "toolUse", - "flue.turn_purpose": "agent", - "model": "gpt-5.4-nano", - "provider": "openai" - }, - "metrics": { - "completion_tokens": 59, - "duration_ms": 0, - "estimated_cost": 0.0004814700000000001, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 1536, - "prompt_tokens": 1885, - "tokens": 3480 - } - }, - { - "name": "tool:bash", - "type": "tool", - "children": [], - "input": { - "command": "printf SKILL_DONE", - "timeout": 100000 - }, - "output": { - "content": [ - { - "text": "SKILL_DONE", - "type": "text" - } - ], - "details": { - "command": "printf SKILL_DONE", - "exitCode": 0 - } - }, - "metadata": { - "flue.session": "task:skill:", - "flue.tool_name": "bash", - "provider": "flue" - }, - "metrics": { - "duration_ms": 0 - } - }, - { - "name": "flue.turn", - "type": "llm", - "children": [], - "input": [ - { - "content": [ - { - "arguments": { - "command": "printf SKILL_DONE", - "timeout": 100000 - }, - "id": "", - "name": "bash", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "SKILL_DONE", - "type": "text" - } - ], - "isError": false, - "role": "toolResult", - "toolCallId": "", - "toolName": "bash" - } - ], - "output": { - "content": [ - { - "text": "SKILL_DONE", - "textSignature": "{\"v\":1,\"id\":\"msg_054a16a2f2ea3217016a705b70a41481a38ece40ecddf69512\",\"phase\":\"final_answer\"}", - "type": "text" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "openai-responses", - "flue.input_message_offset": 17, - "flue.input_mode": "delta", - "flue.model": "gpt-5.4-nano", - "flue.provider": "openai", - "flue.session": "task:skill:", - "flue.stop_reason": "stop", - "flue.turn_purpose": "agent", - "model": "gpt-5.4-nano", - "provider": "openai" - }, - "metrics": { - "completion_tokens": 7, - "duration_ms": 0, - "estimated_cost": 0.00031035000000000004, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 2560, - "prompt_tokens": 1252, - "tokens": 3819 - } - } - ], - "input": "Run the skill named \"e2e-flue-skill\" with arguments {\"marker\":\"SKILL_DONE\"}. Output only the marker and no extra text when finished.", - "output": "SKILL_DONE", - "metadata": { - "flue.session": "skill", - "provider": "flue" - }, - "metrics": { - "duration_ms": 0 - } - } - ], - "input": [ - { - "content": [ - { - "text": "Run the skill named \"e2e-flue-skill\".\n\nArguments:\n{\n \"marker\": \"SKILL_DONE\"\n}", - "type": "text" - } - ], - "role": "user" - } - ], - "output": { - "content": [ - { - "arguments": { - "cwd": ".", - "description": "Run the deterministic Flue instrumentation e2e test skill e2e-flue-skill.", - "prompt": "Run the skill named \"e2e-flue-skill\" with arguments {\"marker\":\"SKILL_DONE\"}. Output only the marker and no extra text when finished." - }, - "id": "", - "name": "task", - "type": "toolCall" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "openai-responses", - "flue.model": "gpt-5.4-nano", - "flue.provider": "openai", - "flue.session": "skill", - "flue.stop_reason": "toolUse", - "flue.turn_purpose": "agent", - "model": "gpt-5.4-nano", - "provider": "openai" - }, - "metrics": { - "completion_tokens": 109, - "duration_ms": 0, - "estimated_cost": 0.00030565000000000003, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 847, - "tokens": 956 - } - }, - { - "name": "tool:task", - "type": "tool", - "children": [], - "input": { - "cwd": ".", - "description": "Run the deterministic Flue instrumentation e2e test skill e2e-flue-skill.", - "prompt": "Run the skill named \"e2e-flue-skill\" with arguments {\"marker\":\"SKILL_DONE\"}. Output only the marker and no extra text when finished." - }, - "output": { - "content": [ - { - "text": "SKILL_DONE", - "type": "text" - } - ], - "details": { - "cwd": ".", - "messageId": "", - "session": "task:skill:", - "taskId": "" - } - }, - "metadata": { - "flue.session": "skill", - "flue.tool_name": "task", - "provider": "flue" - }, - "metrics": { - "duration_ms": 0 - } - }, - { - "name": "flue.turn", - "type": "llm", - "children": [], - "input": [ - { - "content": [ - { - "text": "Run the skill named \"e2e-flue-skill\".\n\nArguments:\n{\n \"marker\": \"SKILL_DONE\"\n}", - "type": "text" - } - ], - "role": "user" - }, - { - "content": [ - { - "arguments": { - "cwd": ".", - "description": "Run the deterministic Flue instrumentation e2e test skill e2e-flue-skill.", - "prompt": "Run the skill named \"e2e-flue-skill\" with arguments {\"marker\":\"SKILL_DONE\"}. Output only the marker and no extra text when finished." - }, - "id": "", - "name": "task", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "SKILL_DONE", - "type": "text" - } - ], - "isError": false, - "role": "toolResult", - "toolCallId": "", - "toolName": "task" - } - ], - "output": { - "content": [ - { - "text": "SKILL_DONE", - "textSignature": "{\"v\":1,\"id\":\"msg_0f099b3fce0721fd016a705b71558881a38c53a767e09d9e31\",\"phase\":\"final_answer\"}", - "type": "text" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "openai-responses", - "flue.model": "gpt-5.4-nano", - "flue.provider": "openai", - "flue.session": "skill", - "flue.stop_reason": "stop", - "flue.turn_purpose": "agent", - "model": "gpt-5.4-nano", - "provider": "openai" - }, - "metrics": { - "completion_tokens": 7, - "duration_ms": 0, - "estimated_cost": 0.00020255000000000002, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 969, - "tokens": 976 - } - } - ], - "input": [ - { - "content": [ - { - "text": "Run the skill named \"e2e-flue-skill\".\n\nArguments:\n{\n \"marker\": \"SKILL_DONE\"\n}", - "type": "text" - } - ], - "role": "user" - } - ], - "output": "SKILL_DONE", - "metadata": { - "flue.operation": "skill", - "flue.session": "skill", - "flue.workflow_name": "instrumentation", - "provider": "flue", - "scenario": "flue-instrumentation" - }, - "metrics": { - "duration_ms": 0 - } - }, - { - "name": "flue.task", - "type": "task", - "children": [ - { - "name": "flue.turn", - "type": "llm", - "children": [], - "input": [ - { - "content": [ - { - "text": "Reply with exactly TASK_DONE and no other text.", - "type": "text" - } - ], - "role": "user" - } - ], - "output": { - "content": [ - { - "text": "TASK_DONE", - "textSignature": "{\"v\":1,\"id\":\"msg_0f3adfdb1510c165016a705b72843c81a0b30d242da683a882\"}", - "type": "text" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "openai-responses", - "flue.input_mode": "full", - "flue.model": "gpt-4o-mini", - "flue.provider": "openai", - "flue.session": "task:task:", - "flue.stop_reason": "stop", - "flue.turn_purpose": "agent", - "model": "gpt-4o-mini", - "provider": "openai" - }, - "metrics": { - "completion_tokens": 4, - "duration_ms": 0, - "estimated_cost": 0.0001263, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 826, - "tokens": 830 - } - } - ], - "input": "Reply with exactly TASK_DONE and no other text.", - "output": "TASK_DONE", - "metadata": { - "flue.session": "task", - "provider": "flue" - }, - "metrics": { - "duration_ms": 0 - } - }, - { - "name": "flue.compact", - "type": "task", - "children": [ - { - "name": "compaction:manual", - "type": "task", - "children": [ - { - "name": "flue.turn", - "type": "llm", - "children": [], - "input": [ - { - "content": [ - { - "text": "\n[User]: Complete this instrumented research flow. Call exactly one tool per turn and wait for each tool result before choosing the next tool. Step 1: call lookup with query \"flue instrumentation\". Step 2: use the lookup result id as lookupId and call web_search with query \"Braintrust Flue reasoning stream instrumentation\". Step 3: use the first web_search result url and call summarize_source. After summarize_source returns, reply with exactly PROMPT_DONE and no other text.\n\n[Assistant tool calls]: lookup(query=\"flue instrumentation\")\n\n[Tool result]: {\"id\":\"flue-session-2026\",\"query\":\"flue instrumentation\",\"topic\":\"session instrumentation\"}\n\n[Assistant tool calls]: web_search(lookupId=\"flue-session-2026\", query=\"Braintrust Flue reasoning stream instrumentation\")\n\n[Tool result]: {\"lookupId\":\"flue-session-2026\",\"query\":\"Braintrust Flue reasoning stream instrumentation\",\"results\":[{\"title\":\"Flue reasoning stream instrumentation\",\"url\":\"https://example.test/flue/reasoning-streams\"}]}\n\n[Assistant tool calls]: summarize_source(url=\"https://example.test/flue/reasoning-streams\")\n\n[Tool result]: {\"summary\":\"Flue emits reasoning, tool execution, and LLM turn events separately.\",\"url\":\"https://example.test/flue/reasoning-streams\"}\n\n\nThis is the PREFIX of a turn that was too large to keep. The SUFFIX (recent work) is retained.\n\nSummarize the prefix to provide context for the retained suffix:\n\n## Original Request\n[What did the user ask for in this turn?]\n\n## Early Progress\n- [Key decisions and work done in the prefix]\n\n## Context for Suffix\n- [Information needed to understand the retained recent work]\n\nBe concise. Focus on what's needed to understand the kept suffix.", - "type": "text" - } - ], - "role": "user" - } - ], - "output": { - "content": [ - { - "text": "## Original Request\nThe user requested a step-by-step execution of an instrumented research flow involving tool calls to gather information on \"flue instrumentation\" and \"", - "textSignature": "{\"v\":1,\"id\":\"msg_04d28ba03840bb65016a705b7358a081a38e69464057840036\"}", - "type": "text" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "openai-responses", - "flue.model": "gpt-4o-mini", - "flue.provider": "openai", - "flue.session": "main", - "flue.stop_reason": "stop", - "flue.turn_purpose": "compaction_prefix", - "model": "gpt-4o-mini", - "provider": "openai" - }, - "metrics": { - "completion_tokens": 0, - "duration_ms": 0, - "estimated_cost": 0, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 0, - "tokens": 0 - } - } - ], - "input": { - "estimatedTokens": 1229, - "reason": "manual" - }, - "output": { - "messagesAfter": 2, - "messagesBefore": 8 - }, - "metadata": { - "flue.compaction_reason": "manual", - "flue.session": "main", - "provider": "flue" - }, - "metrics": { - "duration_ms": 0, - "messages_after": 2, - "messages_before": 8 - } - } - ], - "input": { - "estimatedTokens": 1229, - "reason": "manual" - }, - "output": { - "completed": true - }, - "metadata": { - "flue.operation": "compact", - "flue.session": "main", - "flue.workflow_name": "instrumentation", - "provider": "flue", - "scenario": "flue-instrumentation" - }, - "metrics": { - "duration_ms": 0 - } - } - ], - "input": { - "metadata": { - "scenario": "flue-instrumentation", - "testRunId": "" - }, - "scenario": "flue-instrumentation" - }, - "output": { - "scenario": "flue-instrumentation", - "status": "done" - }, - "metadata": { - "flue.workflow_name": "instrumentation", - "provider": "flue", - "scenario": "flue-instrumentation" - }, - "metrics": { - "duration_ms": 0 - } - }, - { - "name": "flue.prompt", - "type": "task", - "children": [ - { - "name": "flue.turn", - "type": "llm", - "children": [], - "input": [ - { - "content": [ - { - "text": "Complete this instrumented research flow. Call exactly one tool per turn and wait for each tool result before choosing the next tool. Step 1: call lookup with query \"flue instrumentation\". Step 2: use the lookup result id as lookupId and call web_search with query \"Braintrust Flue reasoning stream instrumentation\". Step 3: use the first web_search result url and call summarize_source. After summarize_source returns, reply with exactly PROMPT_DONE and no other text.", - "type": "text" - } - ], - "role": "user" - } - ], - "output": { - "content": [ - { - "arguments": { - "query": "flue instrumentation" - }, - "id": "", - "name": "lookup", - "type": "toolCall" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "openai-responses", - "flue.input_mode": "full", - "flue.model": "gpt-5.4-nano", - "flue.provider": "openai", - "flue.session": "main", - "flue.stop_reason": "toolUse", - "flue.turn_purpose": "agent", - "model": "gpt-5.4-nano", - "provider": "openai" - }, - "metrics": { - "completion_tokens": 19, - "duration_ms": 0, - "estimated_cost": 0.00022475000000000001, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 1005, - "tokens": 1024 - } - }, - { - "name": "tool:lookup", - "type": "tool", - "children": [], - "input": { - "query": "flue instrumentation" - }, - "output": { - "content": [ - { - "text": "{\"id\":\"flue-session-2026\",\"query\":\"flue instrumentation\",\"topic\":\"session instrumentation\"}", - "type": "text" - } - ], - "details": { - "customTool": "lookup" - } - }, - "metadata": { - "flue.session": "main", - "flue.tool_name": "lookup", - "provider": "flue" - }, - "metrics": { - "duration_ms": 0 - } - }, - { - "name": "flue.turn", - "type": "llm", - "children": [], - "input": [ - { - "content": [ - { - "arguments": { - "query": "flue instrumentation" - }, - "id": "", - "name": "lookup", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "{\"id\":\"flue-session-2026\",\"query\":\"flue instrumentation\",\"topic\":\"session instrumentation\"}", - "type": "text" - } - ], - "isError": false, - "role": "toolResult", - "toolCallId": "", - "toolName": "lookup" - } - ], - "output": { - "content": [ - { - "arguments": { - "lookupId": "flue-session-2026", - "query": "Braintrust Flue reasoning stream instrumentation" - }, - "id": "", - "name": "web_search", - "type": "toolCall" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "openai-responses", - "flue.input_message_offset": 1, - "flue.input_mode": "delta", - "flue.model": "gpt-5.4-nano", - "flue.provider": "openai", - "flue.session": "main", - "flue.stop_reason": "toolUse", - "flue.turn_purpose": "agent", - "model": "gpt-5.4-nano", - "provider": "openai" - }, - "metrics": { - "completion_tokens": 34, - "duration_ms": 0, - "estimated_cost": 0.00025350000000000004, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 1055, - "tokens": 1089 - } - }, - { - "name": "tool:web_search", - "type": "tool", - "children": [], - "input": { - "lookupId": "flue-session-2026", - "query": "Braintrust Flue reasoning stream instrumentation" - }, - "output": { - "content": [ - { - "text": "{\"lookupId\":\"flue-session-2026\",\"query\":\"Braintrust Flue reasoning stream instrumentation\",\"results\":[{\"title\":\"Flue reasoning stream instrumentation\",\"url\":\"https://example.test/flue/reasoning-streams\"}]}", - "type": "text" - } - ], - "details": { - "customTool": "web_search" - } - }, - "metadata": { - "flue.session": "main", - "flue.tool_name": "web_search", - "provider": "flue" - }, - "metrics": { - "duration_ms": 0 - } - }, - { - "name": "flue.turn", - "type": "llm", - "children": [], - "input": [ - { - "content": [ - { - "arguments": { - "lookupId": "flue-session-2026", - "query": "Braintrust Flue reasoning stream instrumentation" - }, - "id": "", - "name": "web_search", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "{\"lookupId\":\"flue-session-2026\",\"query\":\"Braintrust Flue reasoning stream instrumentation\",\"results\":[{\"title\":\"Flue reasoning stream instrumentation\",\"url\":\"https://example.test/flue/reasoning-streams\"}]}", - "type": "text" - } - ], - "isError": false, - "role": "toolResult", - "toolCallId": "", - "toolName": "web_search" - } - ], - "output": { - "content": [ - { - "arguments": { - "url": "https://example.test/flue/reasoning-streams" - }, - "id": "", - "name": "summarize_source", - "type": "toolCall" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "openai-responses", - "flue.input_message_offset": 3, - "flue.input_mode": "delta", - "flue.model": "gpt-5.4-nano", - "flue.provider": "openai", - "flue.session": "main", - "flue.stop_reason": "toolUse", - "flue.turn_purpose": "agent", - "model": "gpt-5.4-nano", - "provider": "openai" - }, - "metrics": { - "completion_tokens": 30, - "duration_ms": 0, - "estimated_cost": 0.00026690000000000004, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 1147, - "tokens": 1177 - } - }, - { - "name": "tool:summarize_source", - "type": "tool", - "children": [], - "input": { - "url": "https://example.test/flue/reasoning-streams" - }, - "output": { - "content": [ - { - "text": "{\"summary\":\"Flue emits reasoning, tool execution, and LLM turn events separately.\",\"url\":\"https://example.test/flue/reasoning-streams\"}", - "type": "text" - } - ], - "details": { - "customTool": "summarize_source" - } - }, - "metadata": { - "flue.session": "main", - "flue.tool_name": "summarize_source", - "provider": "flue" - }, - "metrics": { - "duration_ms": 0 - } - }, - { - "name": "flue.turn", - "type": "llm", - "children": [], - "input": [ - { - "content": [ - { - "arguments": { - "url": "https://example.test/flue/reasoning-streams" - }, - "id": "", - "name": "summarize_source", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "{\"summary\":\"Flue emits reasoning, tool execution, and LLM turn events separately.\",\"url\":\"https://example.test/flue/reasoning-streams\"}", - "type": "text" - } - ], - "isError": false, - "role": "toolResult", - "toolCallId": "", - "toolName": "summarize_source" - } - ], - "output": { - "content": [ - { - "text": "PROMPT_DONE", - "textSignature": "{\"v\":1,\"id\":\"msg_045458052dc10379016a705b62dbd8819189b8287061debf12\",\"phase\":\"final_answer\"}", - "type": "text" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "openai-responses", - "flue.input_message_offset": 5, - "flue.input_mode": "delta", - "flue.model": "gpt-5.4-nano", - "flue.provider": "openai", - "flue.session": "main", - "flue.stop_reason": "stop", - "flue.turn_purpose": "agent", - "model": "gpt-5.4-nano", - "provider": "openai" - }, - "metrics": { - "completion_tokens": 7, - "duration_ms": 0, - "estimated_cost": 0.00025315000000000005, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 1222, - "tokens": 1229 - } - } - ], - "input": [ - { - "content": [ - { - "text": "Complete this instrumented research flow. Call exactly one tool per turn and wait for each tool result before choosing the next tool. Step 1: call lookup with query \"flue instrumentation\". Step 2: use the lookup result id as lookupId and call web_search with query \"Braintrust Flue reasoning stream instrumentation\". Step 3: use the first web_search result url and call summarize_source. After summarize_source returns, reply with exactly PROMPT_DONE and no other text.", - "type": "text" - } - ], - "role": "user" - } - ], - "output": "PROMPT_DONE", - "metadata": { - "flue.operation": "prompt", - "flue.session": "main", - "flue.workflow_name": "instrumentation", - "provider": "flue", - "scenario": "flue-instrumentation" - }, - "metrics": { - "duration_ms": 0 - } - } - ] -} diff --git a/e2e/scenarios/flue-instrumentation/__snapshots__/flue-v0-8-latest-auto-hook.span-tree.txt b/e2e/scenarios/flue-instrumentation/__snapshots__/flue-v0-8-latest-auto-hook.span-tree.txt deleted file mode 100644 index abf057e06..000000000 --- a/e2e/scenarios/flue-instrumentation/__snapshots__/flue-v0-8-latest-auto-hook.span-tree.txt +++ /dev/null @@ -1,1499 +0,0 @@ -span_tree: -├── workflow:instrumentation [task] -│ input: { -│ "metadata": { -│ "scenario": "flue-instrumentation", -│ "testRunId": "" -│ }, -│ "scenario": "flue-instrumentation" -│ } -│ output: { -│ "scenario": "flue-instrumentation", -│ "status": "done" -│ } -│ metadata: { -│ "flue.workflow_name": "instrumentation", -│ "provider": "flue", -│ "scenario": "flue-instrumentation" -│ } -│ metrics: { -│ "duration_ms": 0 -│ } -│ ├── flue.skill [task] -│ │ input: [ -│ │ { -│ │ "content": [ -│ │ { -│ │ "text": "Run the skill named \"e2e-flue-skill\".\n\nArguments:\n{\n \"marker\": \"SKILL_DONE\"\n}", -│ │ "type": "text" -│ │ } -│ │ ], -│ │ "role": "user" -│ │ } -│ │ ] -│ │ output: "SKILL_DONE" -│ │ metadata: { -│ │ "flue.operation": "skill", -│ │ "flue.session": "skill", -│ │ "flue.workflow_name": "instrumentation", -│ │ "provider": "flue", -│ │ "scenario": "flue-instrumentation" -│ │ } -│ │ metrics: { -│ │ "duration_ms": 0 -│ │ } -│ │ ├── flue.turn [llm] -│ │ │ input: [ -│ │ │ { -│ │ │ "content": [ -│ │ │ { -│ │ │ "text": "Run the skill named \"e2e-flue-skill\".\n\nArguments:\n{\n \"marker\": \"SKILL_DONE\"\n}", -│ │ │ "type": "text" -│ │ │ } -│ │ │ ], -│ │ │ "role": "user" -│ │ │ } -│ │ │ ] -│ │ │ output: { -│ │ │ "content": [ -│ │ │ { -│ │ │ "arguments": { -│ │ │ "cwd": ".", -│ │ │ "description": "Run the deterministic Flue instrumentation e2e test skill e2e-flue-skill.", -│ │ │ "prompt": "Run the skill named \"e2e-flue-skill\" with arguments {\"marker\":\"SKILL_DONE\"}. Output only the marker and no extra text when finished." -│ │ │ }, -│ │ │ "id": "", -│ │ │ "name": "task", -│ │ │ "type": "toolCall" -│ │ │ } -│ │ │ ], -│ │ │ "role": "assistant" -│ │ │ } -│ │ │ metadata: { -│ │ │ "flue.api": "openai-responses", -│ │ │ "flue.model": "gpt-5.4-nano", -│ │ │ "flue.provider": "openai", -│ │ │ "flue.session": "skill", -│ │ │ "flue.stop_reason": "toolUse", -│ │ │ "flue.turn_purpose": "agent", -│ │ │ "model": "gpt-5.4-nano", -│ │ │ "provider": "openai" -│ │ │ } -│ │ │ metrics: { -│ │ │ "completion_tokens": 109, -│ │ │ "duration_ms": 0, -│ │ │ "estimated_cost": 0.00030565000000000003, -│ │ │ "prompt_cache_creation_tokens": 0, -│ │ │ "prompt_cached_tokens": 0, -│ │ │ "prompt_tokens": 847, -│ │ │ "tokens": 956 -│ │ │ } -│ │ │ └── flue.task [task] -│ │ │ input: "Run the skill named \"e2e-flue-skill\" with arguments {\"marker\":\"SKILL_DONE\"}. Output only the marker and no extra text when finished." -│ │ │ output: "SKILL_DONE" -│ │ │ metadata: { -│ │ │ "flue.session": "skill", -│ │ │ "provider": "flue" -│ │ │ } -│ │ │ metrics: { -│ │ │ "duration_ms": 0 -│ │ │ } -│ │ │ ├── flue.turn [llm] -│ │ │ │ input: [ -│ │ │ │ { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "text": "Run the skill named \"e2e-flue-skill\" with arguments {\"marker\":\"SKILL_DONE\"}. Output only the marker and no extra text when finished.", -│ │ │ │ "type": "text" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "role": "user" -│ │ │ │ } -│ │ │ │ ] -│ │ │ │ output: { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "arguments": { -│ │ │ │ "agent": "e2e-flue-skill", -│ │ │ │ "cwd": ".", -│ │ │ │ "description": "Run e2e-flue-skill with marker SKILL_DONE", -│ │ │ │ "prompt": "Run the skill named \"e2e-flue-skill\" with arguments {\"marker\":\"SKILL_DONE\"}. Ensure completion and report the final output marker only." -│ │ │ │ }, -│ │ │ │ "id": "", -│ │ │ │ "name": "task", -│ │ │ │ "type": "toolCall" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "role": "assistant" -│ │ │ │ } -│ │ │ │ metadata: { -│ │ │ │ "flue.api": "openai-responses", -│ │ │ │ "flue.input_mode": "full", -│ │ │ │ "flue.model": "gpt-5.4-nano", -│ │ │ │ "flue.provider": "openai", -│ │ │ │ "flue.session": "task:skill:", -│ │ │ │ "flue.stop_reason": "toolUse", -│ │ │ │ "flue.turn_purpose": "agent", -│ │ │ │ "model": "gpt-5.4-nano", -│ │ │ │ "provider": "openai" -│ │ │ │ } -│ │ │ │ metrics: { -│ │ │ │ "completion_tokens": 113, -│ │ │ │ "duration_ms": 0, -│ │ │ │ "estimated_cost": 0.00031225000000000003, -│ │ │ │ "prompt_cache_creation_tokens": 0, -│ │ │ │ "prompt_cached_tokens": 0, -│ │ │ │ "prompt_tokens": 855, -│ │ │ │ "tokens": 968 -│ │ │ │ } -│ │ │ ├── tool:task [tool] -│ │ │ │ input: { -│ │ │ │ "agent": "e2e-flue-skill", -│ │ │ │ "cwd": ".", -│ │ │ │ "description": "Run e2e-flue-skill with marker SKILL_DONE", -│ │ │ │ "prompt": "Run the skill named \"e2e-flue-skill\" with arguments {\"marker\":\"SKILL_DONE\"}. Ensure completion and report the final output marker only." -│ │ │ │ } -│ │ │ │ output: { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "text": "[flue] Subagent \"e2e-flue-skill\" is not declared. Available: (none).", -│ │ │ │ "type": "text" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "details": {} -│ │ │ │ } -│ │ │ │ metadata: { -│ │ │ │ "flue.session": "task:skill:", -│ │ │ │ "flue.tool_name": "task", -│ │ │ │ "provider": "flue" -│ │ │ │ } -│ │ │ │ metrics: { -│ │ │ │ "duration_ms": 0 -│ │ │ │ } -│ │ │ │ error: "{\"content\":[{\"type\":\"text\",\"text\":\"[flue] Subagent \\\"e2e-flue-skill\\\" is not declared. Available: (none).\"}],\"details\":{}}" -│ │ │ ├── flue.turn [llm] -│ │ │ │ input: [ -│ │ │ │ { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "arguments": { -│ │ │ │ "agent": "e2e-flue-skill", -│ │ │ │ "cwd": ".", -│ │ │ │ "description": "Run e2e-flue-skill with marker SKILL_DONE", -│ │ │ │ "prompt": "Run the skill named \"e2e-flue-skill\" with arguments {\"marker\":\"SKILL_DONE\"}. Ensure completion and report the final output marker only." -│ │ │ │ }, -│ │ │ │ "id": "", -│ │ │ │ "name": "task", -│ │ │ │ "type": "toolCall" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "role": "assistant" -│ │ │ │ }, -│ │ │ │ { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "text": "[flue] Subagent \"e2e-flue-skill\" is not declared. Available: (none).", -│ │ │ │ "type": "text" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "isError": true, -│ │ │ │ "role": "toolResult", -│ │ │ │ "toolCallId": "", -│ │ │ │ "toolName": "task" -│ │ │ │ } -│ │ │ │ ] -│ │ │ │ output: { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "arguments": { -│ │ │ │ "command": "find . -maxdepth 4 -name AGENTS.md -print", -│ │ │ │ "timeout": 100000 -│ │ │ │ }, -│ │ │ │ "id": "", -│ │ │ │ "name": "bash", -│ │ │ │ "type": "toolCall" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "role": "assistant" -│ │ │ │ } -│ │ │ │ metadata: { -│ │ │ │ "flue.api": "openai-responses", -│ │ │ │ "flue.input_message_offset": 1, -│ │ │ │ "flue.input_mode": "delta", -│ │ │ │ "flue.model": "gpt-5.4-nano", -│ │ │ │ "flue.provider": "openai", -│ │ │ │ "flue.session": "task:skill:", -│ │ │ │ "flue.stop_reason": "toolUse", -│ │ │ │ "flue.turn_purpose": "agent", -│ │ │ │ "model": "gpt-5.4-nano", -│ │ │ │ "provider": "openai" -│ │ │ │ } -│ │ │ │ metrics: { -│ │ │ │ "completion_tokens": 94, -│ │ │ │ "duration_ms": 0, -│ │ │ │ "estimated_cost": 0.00031790000000000003, -│ │ │ │ "prompt_cache_creation_tokens": 0, -│ │ │ │ "prompt_cached_tokens": 0, -│ │ │ │ "prompt_tokens": 1002, -│ │ │ │ "tokens": 1096 -│ │ │ │ } -│ │ │ ├── tool:bash [tool] -│ │ │ │ input: { -│ │ │ │ "command": "find . -maxdepth 4 -name AGENTS.md -print", -│ │ │ │ "timeout": 100000 -│ │ │ │ } -│ │ │ │ output: { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "text": "(no output)", -│ │ │ │ "type": "text" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "details": { -│ │ │ │ "command": "find . -maxdepth 4 -name AGENTS.md -print", -│ │ │ │ "exitCode": 0 -│ │ │ │ } -│ │ │ │ } -│ │ │ │ metadata: { -│ │ │ │ "flue.session": "task:skill:", -│ │ │ │ "flue.tool_name": "bash", -│ │ │ │ "provider": "flue" -│ │ │ │ } -│ │ │ │ metrics: { -│ │ │ │ "duration_ms": 0 -│ │ │ │ } -│ │ │ ├── flue.turn [llm] -│ │ │ │ input: [ -│ │ │ │ { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "arguments": { -│ │ │ │ "command": "find . -maxdepth 4 -name AGENTS.md -print", -│ │ │ │ "timeout": 100000 -│ │ │ │ }, -│ │ │ │ "id": "", -│ │ │ │ "name": "bash", -│ │ │ │ "type": "toolCall" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "role": "assistant" -│ │ │ │ }, -│ │ │ │ { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "text": "(no output)", -│ │ │ │ "type": "text" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "isError": false, -│ │ │ │ "role": "toolResult", -│ │ │ │ "toolCallId": "", -│ │ │ │ "toolName": "bash" -│ │ │ │ } -│ │ │ │ ] -│ │ │ │ output: { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "arguments": { -│ │ │ │ "command": "find . -maxdepth 4 -type f -name AGENTS.md -print -exec sed -n '1,200p' {} \\;", -│ │ │ │ "timeout": 100000 -│ │ │ │ }, -│ │ │ │ "id": "", -│ │ │ │ "name": "bash", -│ │ │ │ "type": "toolCall" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "role": "assistant" -│ │ │ │ } -│ │ │ │ metadata: { -│ │ │ │ "flue.api": "openai-responses", -│ │ │ │ "flue.input_message_offset": 3, -│ │ │ │ "flue.input_mode": "delta", -│ │ │ │ "flue.model": "gpt-5.4-nano", -│ │ │ │ "flue.provider": "openai", -│ │ │ │ "flue.session": "task:skill:", -│ │ │ │ "flue.stop_reason": "toolUse", -│ │ │ │ "flue.turn_purpose": "agent", -│ │ │ │ "model": "gpt-5.4-nano", -│ │ │ │ "provider": "openai" -│ │ │ │ } -│ │ │ │ metrics: { -│ │ │ │ "completion_tokens": 87, -│ │ │ │ "duration_ms": 0, -│ │ │ │ "estimated_cost": 0.00034815000000000003, -│ │ │ │ "prompt_cache_creation_tokens": 0, -│ │ │ │ "prompt_cached_tokens": 0, -│ │ │ │ "prompt_tokens": 1197, -│ │ │ │ "tokens": 1284 -│ │ │ │ } -│ │ │ ├── tool:bash [tool] -│ │ │ │ input: { -│ │ │ │ "command": "find . -maxdepth 4 -type f -name AGENTS.md -print -exec sed -n '1,200p' {} \\;", -│ │ │ │ "timeout": 100000 -│ │ │ │ } -│ │ │ │ output: { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "text": "(no output)", -│ │ │ │ "type": "text" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "details": { -│ │ │ │ "command": "find . -maxdepth 4 -type f -name AGENTS.md -print -exec sed -n '1,200p' {} \\;", -│ │ │ │ "exitCode": 0 -│ │ │ │ } -│ │ │ │ } -│ │ │ │ metadata: { -│ │ │ │ "flue.session": "task:skill:", -│ │ │ │ "flue.tool_name": "bash", -│ │ │ │ "provider": "flue" -│ │ │ │ } -│ │ │ │ metrics: { -│ │ │ │ "duration_ms": 0 -│ │ │ │ } -│ │ │ ├── flue.turn [llm] -│ │ │ │ input: [ -│ │ │ │ { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "arguments": { -│ │ │ │ "command": "find . -maxdepth 4 -type f -name AGENTS.md -print -exec sed -n '1,200p' {} \\;", -│ │ │ │ "timeout": 100000 -│ │ │ │ }, -│ │ │ │ "id": "", -│ │ │ │ "name": "bash", -│ │ │ │ "type": "toolCall" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "role": "assistant" -│ │ │ │ }, -│ │ │ │ { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "text": "(no output)", -│ │ │ │ "type": "text" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "isError": false, -│ │ │ │ "role": "toolResult", -│ │ │ │ "toolCallId": "", -│ │ │ │ "toolName": "bash" -│ │ │ │ } -│ │ │ │ ] -│ │ │ │ output: { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "arguments": { -│ │ │ │ "command": "find . -maxdepth 6 -type f -name 'AGENTS.md' -print", -│ │ │ │ "timeout": 100000 -│ │ │ │ }, -│ │ │ │ "id": "", -│ │ │ │ "name": "bash", -│ │ │ │ "type": "toolCall" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "role": "assistant" -│ │ │ │ } -│ │ │ │ metadata: { -│ │ │ │ "flue.api": "openai-responses", -│ │ │ │ "flue.input_message_offset": 5, -│ │ │ │ "flue.input_mode": "delta", -│ │ │ │ "flue.model": "gpt-5.4-nano", -│ │ │ │ "flue.provider": "openai", -│ │ │ │ "flue.session": "task:skill:", -│ │ │ │ "flue.stop_reason": "toolUse", -│ │ │ │ "flue.turn_purpose": "agent", -│ │ │ │ "model": "gpt-5.4-nano", -│ │ │ │ "provider": "openai" -│ │ │ │ } -│ │ │ │ metrics: { -│ │ │ │ "completion_tokens": 58, -│ │ │ │ "duration_ms": 0, -│ │ │ │ "estimated_cost": 0.0003321, -│ │ │ │ "prompt_cache_creation_tokens": 0, -│ │ │ │ "prompt_cached_tokens": 0, -│ │ │ │ "prompt_tokens": 1298, -│ │ │ │ "tokens": 1356 -│ │ │ │ } -│ │ │ ├── tool:bash [tool] -│ │ │ │ input: { -│ │ │ │ "command": "find . -maxdepth 6 -type f -name 'AGENTS.md' -print", -│ │ │ │ "timeout": 100000 -│ │ │ │ } -│ │ │ │ output: { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "text": "(no output)", -│ │ │ │ "type": "text" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "details": { -│ │ │ │ "command": "find . -maxdepth 6 -type f -name 'AGENTS.md' -print", -│ │ │ │ "exitCode": 0 -│ │ │ │ } -│ │ │ │ } -│ │ │ │ metadata: { -│ │ │ │ "flue.session": "task:skill:", -│ │ │ │ "flue.tool_name": "bash", -│ │ │ │ "provider": "flue" -│ │ │ │ } -│ │ │ │ metrics: { -│ │ │ │ "duration_ms": 0 -│ │ │ │ } -│ │ │ ├── flue.turn [llm] -│ │ │ │ input: [ -│ │ │ │ { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "arguments": { -│ │ │ │ "command": "find . -maxdepth 6 -type f -name 'AGENTS.md' -print", -│ │ │ │ "timeout": 100000 -│ │ │ │ }, -│ │ │ │ "id": "", -│ │ │ │ "name": "bash", -│ │ │ │ "type": "toolCall" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "role": "assistant" -│ │ │ │ }, -│ │ │ │ { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "text": "(no output)", -│ │ │ │ "type": "text" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "isError": false, -│ │ │ │ "role": "toolResult", -│ │ │ │ "toolCallId": "", -│ │ │ │ "toolName": "bash" -│ │ │ │ } -│ │ │ │ ] -│ │ │ │ output: { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "arguments": { -│ │ │ │ "command": "find . -maxdepth 5 -type f -name '*flue*skill*' -o -name '*skill*' | head", -│ │ │ │ "timeout": 100000 -│ │ │ │ }, -│ │ │ │ "id": "", -│ │ │ │ "name": "bash", -│ │ │ │ "type": "toolCall" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "role": "assistant" -│ │ │ │ } -│ │ │ │ metadata: { -│ │ │ │ "flue.api": "openai-responses", -│ │ │ │ "flue.input_message_offset": 7, -│ │ │ │ "flue.input_mode": "delta", -│ │ │ │ "flue.model": "gpt-5.4-nano", -│ │ │ │ "flue.provider": "openai", -│ │ │ │ "flue.session": "task:skill:", -│ │ │ │ "flue.stop_reason": "toolUse", -│ │ │ │ "flue.turn_purpose": "agent", -│ │ │ │ "model": "gpt-5.4-nano", -│ │ │ │ "provider": "openai" -│ │ │ │ } -│ │ │ │ metrics: { -│ │ │ │ "completion_tokens": 68, -│ │ │ │ "duration_ms": 0, -│ │ │ │ "estimated_cost": 0.00035900000000000005, -│ │ │ │ "prompt_cache_creation_tokens": 0, -│ │ │ │ "prompt_cached_tokens": 0, -│ │ │ │ "prompt_tokens": 1370, -│ │ │ │ "tokens": 1438 -│ │ │ │ } -│ │ │ ├── tool:bash [tool] -│ │ │ │ input: { -│ │ │ │ "command": "find . -maxdepth 5 -type f -name '*flue*skill*' -o -name '*skill*' | head", -│ │ │ │ "timeout": 100000 -│ │ │ │ } -│ │ │ │ output: { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "text": "./.agents/skills\n./.agents/skills/e2e-flue-skill", -│ │ │ │ "type": "text" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "details": { -│ │ │ │ "command": "find . -maxdepth 5 -type f -name '*flue*skill*' -o -name '*skill*' | head", -│ │ │ │ "exitCode": 0 -│ │ │ │ } -│ │ │ │ } -│ │ │ │ metadata: { -│ │ │ │ "flue.session": "task:skill:", -│ │ │ │ "flue.tool_name": "bash", -│ │ │ │ "provider": "flue" -│ │ │ │ } -│ │ │ │ metrics: { -│ │ │ │ "duration_ms": 0 -│ │ │ │ } -│ │ │ ├── flue.turn [llm] -│ │ │ │ input: [ -│ │ │ │ { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "arguments": { -│ │ │ │ "command": "find . -maxdepth 5 -type f -name '*flue*skill*' -o -name '*skill*' | head", -│ │ │ │ "timeout": 100000 -│ │ │ │ }, -│ │ │ │ "id": "", -│ │ │ │ "name": "bash", -│ │ │ │ "type": "toolCall" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "role": "assistant" -│ │ │ │ }, -│ │ │ │ { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "text": "./.agents/skills\n./.agents/skills/e2e-flue-skill", -│ │ │ │ "type": "text" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "isError": false, -│ │ │ │ "role": "toolResult", -│ │ │ │ "toolCallId": "", -│ │ │ │ "toolName": "bash" -│ │ │ │ } -│ │ │ │ ] -│ │ │ │ output: { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "arguments": { -│ │ │ │ "command": "find ./.agents/skills/e2e-flue-skill -maxdepth 2 -type f -print", -│ │ │ │ "timeout": 100000 -│ │ │ │ }, -│ │ │ │ "id": "", -│ │ │ │ "name": "bash", -│ │ │ │ "type": "toolCall" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "role": "assistant" -│ │ │ │ } -│ │ │ │ metadata: { -│ │ │ │ "flue.api": "openai-responses", -│ │ │ │ "flue.input_message_offset": 9, -│ │ │ │ "flue.input_mode": "delta", -│ │ │ │ "flue.model": "gpt-5.4-nano", -│ │ │ │ "flue.provider": "openai", -│ │ │ │ "flue.session": "task:skill:", -│ │ │ │ "flue.stop_reason": "toolUse", -│ │ │ │ "flue.turn_purpose": "agent", -│ │ │ │ "model": "gpt-5.4-nano", -│ │ │ │ "provider": "openai" -│ │ │ │ } -│ │ │ │ metrics: { -│ │ │ │ "completion_tokens": 109, -│ │ │ │ "duration_ms": 0, -│ │ │ │ "estimated_cost": 0.00043885000000000007, -│ │ │ │ "prompt_cache_creation_tokens": 0, -│ │ │ │ "prompt_cached_tokens": 0, -│ │ │ │ "prompt_tokens": 1513, -│ │ │ │ "tokens": 1622 -│ │ │ │ } -│ │ │ ├── tool:bash [tool] -│ │ │ │ input: { -│ │ │ │ "command": "find ./.agents/skills/e2e-flue-skill -maxdepth 2 -type f -print", -│ │ │ │ "timeout": 100000 -│ │ │ │ } -│ │ │ │ output: { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "text": "./.agents/skills/e2e-flue-skill/SKILL.md", -│ │ │ │ "type": "text" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "details": { -│ │ │ │ "command": "find ./.agents/skills/e2e-flue-skill -maxdepth 2 -type f -print", -│ │ │ │ "exitCode": 0 -│ │ │ │ } -│ │ │ │ } -│ │ │ │ metadata: { -│ │ │ │ "flue.session": "task:skill:", -│ │ │ │ "flue.tool_name": "bash", -│ │ │ │ "provider": "flue" -│ │ │ │ } -│ │ │ │ metrics: { -│ │ │ │ "duration_ms": 0 -│ │ │ │ } -│ │ │ ├── flue.turn [llm] -│ │ │ │ input: [ -│ │ │ │ { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "arguments": { -│ │ │ │ "command": "find ./.agents/skills/e2e-flue-skill -maxdepth 2 -type f -print", -│ │ │ │ "timeout": 100000 -│ │ │ │ }, -│ │ │ │ "id": "", -│ │ │ │ "name": "bash", -│ │ │ │ "type": "toolCall" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "role": "assistant" -│ │ │ │ }, -│ │ │ │ { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "text": "./.agents/skills/e2e-flue-skill/SKILL.md", -│ │ │ │ "type": "text" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "isError": false, -│ │ │ │ "role": "toolResult", -│ │ │ │ "toolCallId": "", -│ │ │ │ "toolName": "bash" -│ │ │ │ } -│ │ │ │ ] -│ │ │ │ output: { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "arguments": { -│ │ │ │ "limit": 200, -│ │ │ │ "offset": 1, -│ │ │ │ "path": "./.agents/skills/e2e-flue-skill/SKILL.md" -│ │ │ │ }, -│ │ │ │ "id": "", -│ │ │ │ "name": "read", -│ │ │ │ "type": "toolCall" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "role": "assistant" -│ │ │ │ } -│ │ │ │ metadata: { -│ │ │ │ "flue.api": "openai-responses", -│ │ │ │ "flue.input_message_offset": 11, -│ │ │ │ "flue.input_mode": "delta", -│ │ │ │ "flue.model": "gpt-5.4-nano", -│ │ │ │ "flue.provider": "openai", -│ │ │ │ "flue.session": "task:skill:", -│ │ │ │ "flue.stop_reason": "toolUse", -│ │ │ │ "flue.turn_purpose": "agent", -│ │ │ │ "model": "gpt-5.4-nano", -│ │ │ │ "provider": "openai" -│ │ │ │ } -│ │ │ │ metrics: { -│ │ │ │ "completion_tokens": 64, -│ │ │ │ "duration_ms": 0, -│ │ │ │ "estimated_cost": 0.00015592, -│ │ │ │ "prompt_cache_creation_tokens": 0, -│ │ │ │ "prompt_cached_tokens": 1536, -│ │ │ │ "prompt_tokens": 226, -│ │ │ │ "tokens": 1826 -│ │ │ │ } -│ │ │ ├── tool:read [tool] -│ │ │ │ input: { -│ │ │ │ "limit": 200, -│ │ │ │ "offset": 1, -│ │ │ │ "path": "./.agents/skills/e2e-flue-skill/SKILL.md" -│ │ │ │ } -│ │ │ │ output: { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "text": "---\nname: e2e-flue-skill\ndescription: Deterministic skill used by the Braintrust Flue instrumentation e2e test.\n---\n\nReturn the marker from the provided args exactly once. Output no other text.\n", -│ │ │ │ "type": "text" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "details": { -│ │ │ │ "lines": 7, -│ │ │ │ "path": "./.agents/skills/e2e-flue-skill/SKILL.md" -│ │ │ │ } -│ │ │ │ } -│ │ │ │ metadata: { -│ │ │ │ "flue.session": "task:skill:", -│ │ │ │ "flue.tool_name": "read", -│ │ │ │ "provider": "flue" -│ │ │ │ } -│ │ │ │ metrics: { -│ │ │ │ "duration_ms": 0 -│ │ │ │ } -│ │ │ ├── flue.turn [llm] -│ │ │ │ input: [ -│ │ │ │ { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "arguments": { -│ │ │ │ "limit": 200, -│ │ │ │ "offset": 1, -│ │ │ │ "path": "./.agents/skills/e2e-flue-skill/SKILL.md" -│ │ │ │ }, -│ │ │ │ "id": "", -│ │ │ │ "name": "read", -│ │ │ │ "type": "toolCall" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "role": "assistant" -│ │ │ │ }, -│ │ │ │ { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "text": "---\nname: e2e-flue-skill\ndescription: Deterministic skill used by the Braintrust Flue instrumentation e2e test.\n---\n\nReturn the marker from the provided args exactly once. Output no other text.\n", -│ │ │ │ "type": "text" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "isError": false, -│ │ │ │ "role": "toolResult", -│ │ │ │ "toolCallId": "", -│ │ │ │ "toolName": "read" -│ │ │ │ } -│ │ │ │ ] -│ │ │ │ output: { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "arguments": { -│ │ │ │ "command": "cat ./package.json", -│ │ │ │ "timeout": 100000 -│ │ │ │ }, -│ │ │ │ "id": "", -│ │ │ │ "name": "bash", -│ │ │ │ "type": "toolCall" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "role": "assistant" -│ │ │ │ } -│ │ │ │ metadata: { -│ │ │ │ "flue.api": "openai-responses", -│ │ │ │ "flue.input_message_offset": 13, -│ │ │ │ "flue.input_mode": "delta", -│ │ │ │ "flue.model": "gpt-5.4-nano", -│ │ │ │ "flue.provider": "openai", -│ │ │ │ "flue.session": "task:skill:", -│ │ │ │ "flue.stop_reason": "toolUse", -│ │ │ │ "flue.turn_purpose": "agent", -│ │ │ │ "model": "gpt-5.4-nano", -│ │ │ │ "provider": "openai" -│ │ │ │ } -│ │ │ │ metrics: { -│ │ │ │ "completion_tokens": 90, -│ │ │ │ "duration_ms": 0, -│ │ │ │ "estimated_cost": 0.00021242000000000003, -│ │ │ │ "prompt_cache_creation_tokens": 0, -│ │ │ │ "prompt_cached_tokens": 1536, -│ │ │ │ "prompt_tokens": 346, -│ │ │ │ "tokens": 1972 -│ │ │ │ } -│ │ │ ├── tool:bash [tool] -│ │ │ │ input: { -│ │ │ │ "command": "cat ./package.json", -│ │ │ │ "timeout": 100000 -│ │ │ │ } -│ │ │ │ output: { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "text": "{\n \"name\": \"flue-instrumentation-scenario\",\n \"private\": true,\n \"type\": \"module\",\n \"braintrustScenario\": {\n \"bump\": {\n \"dependencies\": {\n \"flue-cli-v0-8-latest\": {\n \"package\": \"@flue/cli\",\n \"range\": \">=0.8.0 <0.9.0\"\n },\n \"flue-cli-v1-latest\": {\n \"package\": \"@flue/cli\",\n \"range\": \">=1.0.0-beta.3 <2.0.0-0\",\n \"allowPrerelease\": true\n },\n \"flue-runtime-v0-8-latest\": {\n \"package\": \"@flue/runtime\",\n \"range\": \">=0.8.0 <0.9.0\"\n },\n \"flue-runtime-v1-latest\": {\n \"package\": \"@flue/runtime\",\n \"range\": \">=1.0.0-beta.3 <2.0.0-0\",\n \"allowPrerelease\": true\n },\n \"flue-runtime-v2-latest\": {\n \"package\": \"@flue/runtime\",\n \"range\": \">=2.0.0 <3.0.0\"\n }\n }\n }\n },\n \"dependencies\": {\n \"@flue/cli\": \"0.8.0\",\n \"@flue/runtime\": \"0.8.0\",\n \"flue-cli-v0-8-latest\": \"npm:@flue/cli@0.8.1\",\n \"flue-cli-v1\": \"npm:@flue/cli@1.0.0-beta.3\",\n \"flue-cli-v1-latest\": \"npm:@flue/cli@1.0.0-beta.9\",\n \"flue-runtime-v0-8-latest\": \"npm:@flue/runtime@0.8.1\",\n \"flue-runtime-v1\": \"npm:@flue/runtime@1.0.0-beta.3\",\n \"flue-runtime-v1-latest\": \"npm:@flue/runtime@1.0.0-beta.9\",\n \"flue-runtime-v2\": \"npm:@flue/runtime@2.0.0\",\n \"flue-runtime-v2-latest\": \"npm:@flue/runtime@2.0.1\",\n \"pi-ai-v2\": \"npm:@earendil-works/pi-ai@0.83.0\",\n \"valibot\": \"1.4.2\"\n },\n \"pnpm\": {\n \"overrides\": {\n \"@flue/runtime@0.8.0>@earendil-works/pi-agent-core\": \"0.75.4\",\n \"@flue/runtime@0.8.0>@earendil-works/pi-ai\": \"0.75.4\",\n \"@flue/runtime@0.8.1>@earendil-works/pi-agent-core\": \"0.75.4\",\n \"@flue/runtime@0.8.1>@earendil-works/pi-ai\": \"0.75.4\",\n \"@hono/node-server@>=2.0.0 <2.0.10\": \"^2.0.10\",\n \"fast-uri@<3.1.5\": \"^3.1.5\",\n \"hono@>=4.0.0 <4.12.27\": \"^4.12.27\",\n \"brace-expansion@>=3.0.0 <5.0.9\": \"^5.0.9\",\n \"postcss@<8.5.18\": \"^8.5.18\",\n \"valibot@<1.4.2\": \"^1.4.2\",\n \"ip-address@<10.2.2\": \"^10.2.2\",\n \"undici@<7.29.0\": \"^7.29.0\"\n }\n }\n}", -│ │ │ │ "type": "text" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "details": { -│ │ │ │ "command": "cat ./package.json", -│ │ │ │ "exitCode": 0 -│ │ │ │ } -│ │ │ │ } -│ │ │ │ metadata: { -│ │ │ │ "flue.session": "task:skill:", -│ │ │ │ "flue.tool_name": "bash", -│ │ │ │ "provider": "flue" -│ │ │ │ } -│ │ │ │ metrics: { -│ │ │ │ "duration_ms": 0 -│ │ │ │ } -│ │ │ ├── flue.turn [llm] -│ │ │ │ input: [ -│ │ │ │ { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "arguments": { -│ │ │ │ "command": "cat ./package.json", -│ │ │ │ "timeout": 100000 -│ │ │ │ }, -│ │ │ │ "id": "", -│ │ │ │ "name": "bash", -│ │ │ │ "type": "toolCall" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "role": "assistant" -│ │ │ │ }, -│ │ │ │ { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "text": "{\n \"name\": \"flue-instrumentation-scenario\",\n \"private\": true,\n \"type\": \"module\",\n \"braintrustScenario\": {\n \"bump\": {\n \"dependencies\": {\n \"flue-cli-v0-8-latest\": {\n \"package\": \"@flue/cli\",\n \"range\": \">=0.8.0 <0.9.0\"\n },\n \"flue-cli-v1-latest\": {\n \"package\": \"@flue/cli\",\n \"range\": \">=1.0.0-beta.3 <2.0.0-0\",\n \"allowPrerelease\": true\n },\n \"flue-runtime-v0-8-latest\": {\n \"package\": \"@flue/runtime\",\n \"range\": \">=0.8.0 <0.9.0\"\n },\n \"flue-runtime-v1-latest\": {\n \"package\": \"@flue/runtime\",\n \"range\": \">=1.0.0-beta.3 <2.0.0-0\",\n \"allowPrerelease\": true\n },\n \"flue-runtime-v2-latest\": {\n \"package\": \"@flue/runtime\",\n \"range\": \">=2.0.0 <3.0.0\"\n }\n }\n }\n },\n \"dependencies\": {\n \"@flue/cli\": \"0.8.0\",\n \"@flue/runtime\": \"0.8.0\",\n \"flue-cli-v0-8-latest\": \"npm:@flue/cli@0.8.1\",\n \"flue-cli-v1\": \"npm:@flue/cli@1.0.0-beta.3\",\n \"flue-cli-v1-latest\": \"npm:@flue/cli@1.0.0-beta.9\",\n \"flue-runtime-v0-8-latest\": \"npm:@flue/runtime@0.8.1\",\n \"flue-runtime-v1\": \"npm:@flue/runtime@1.0.0-beta.3\",\n \"flue-runtime-v1-latest\": \"npm:@flue/runtime@1.0.0-beta.9\",\n \"flue-runtime-v2\": \"npm:@flue/runtime@2.0.0\",\n \"flue-runtime-v2-latest\": \"npm:@flue/runtime@2.0.1\",\n \"pi-ai-v2\": \"npm:@earendil-works/pi-ai@0.83.0\",\n \"valibot\": \"1.4.2\"\n },\n \"pnpm\": {\n \"overrides\": {\n \"@flue/runtime@0.8.0>@earendil-works/pi-agent-core\": \"0.75.4\",\n \"@flue/runtime@0.8.0>@earendil-works/pi-ai\": \"0.75.4\",\n \"@flue/runtime@0.8.1>@earendil-works/pi-agent-core\": \"0.75.4\",\n \"@flue/runtime@0.8.1>@earendil-works/pi-ai\": \"0.75.4\",\n \"@hono/node-server@>=2.0.0 <2.0.10\": \"^2.0.10\",\n \"fast-uri@<3.1.5\": \"^3.1.5\",\n \"hono@>=4.0.0 <4.12.27\": \"^4.12.27\",\n \"brace-expansion@>=3.0.0 <5.0.9\": \"^5.0.9\",\n \"postcss@<8.5.18\": \"^8.5.18\",\n \"valibot@<1.4.2\": \"^1.4.2\",\n \"ip-address@<10.2.2\": \"^10.2.2\",\n \"undici@<7.29.0\": \"^7.29.0\"\n }\n }\n}", -│ │ │ │ "type": "text" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "isError": false, -│ │ │ │ "role": "toolResult", -│ │ │ │ "toolCallId": "", -│ │ │ │ "toolName": "bash" -│ │ │ │ } -│ │ │ │ ] -│ │ │ │ output: { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "arguments": { -│ │ │ │ "command": "printf SKILL_DONE", -│ │ │ │ "timeout": 100000 -│ │ │ │ }, -│ │ │ │ "id": "", -│ │ │ │ "name": "bash", -│ │ │ │ "type": "toolCall" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "role": "assistant" -│ │ │ │ } -│ │ │ │ metadata: { -│ │ │ │ "flue.api": "openai-responses", -│ │ │ │ "flue.input_message_offset": 15, -│ │ │ │ "flue.input_mode": "delta", -│ │ │ │ "flue.model": "gpt-5.4-nano", -│ │ │ │ "flue.provider": "openai", -│ │ │ │ "flue.session": "task:skill:", -│ │ │ │ "flue.stop_reason": "toolUse", -│ │ │ │ "flue.turn_purpose": "agent", -│ │ │ │ "model": "gpt-5.4-nano", -│ │ │ │ "provider": "openai" -│ │ │ │ } -│ │ │ │ metrics: { -│ │ │ │ "completion_tokens": 59, -│ │ │ │ "duration_ms": 0, -│ │ │ │ "estimated_cost": 0.0004814700000000001, -│ │ │ │ "prompt_cache_creation_tokens": 0, -│ │ │ │ "prompt_cached_tokens": 1536, -│ │ │ │ "prompt_tokens": 1885, -│ │ │ │ "tokens": 3480 -│ │ │ │ } -│ │ │ ├── tool:bash [tool] -│ │ │ │ input: { -│ │ │ │ "command": "printf SKILL_DONE", -│ │ │ │ "timeout": 100000 -│ │ │ │ } -│ │ │ │ output: { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "text": "SKILL_DONE", -│ │ │ │ "type": "text" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "details": { -│ │ │ │ "command": "printf SKILL_DONE", -│ │ │ │ "exitCode": 0 -│ │ │ │ } -│ │ │ │ } -│ │ │ │ metadata: { -│ │ │ │ "flue.session": "task:skill:", -│ │ │ │ "flue.tool_name": "bash", -│ │ │ │ "provider": "flue" -│ │ │ │ } -│ │ │ │ metrics: { -│ │ │ │ "duration_ms": 0 -│ │ │ │ } -│ │ │ └── flue.turn [llm] -│ │ │ input: [ -│ │ │ { -│ │ │ "content": [ -│ │ │ { -│ │ │ "arguments": { -│ │ │ "command": "printf SKILL_DONE", -│ │ │ "timeout": 100000 -│ │ │ }, -│ │ │ "id": "", -│ │ │ "name": "bash", -│ │ │ "type": "toolCall" -│ │ │ } -│ │ │ ], -│ │ │ "role": "assistant" -│ │ │ }, -│ │ │ { -│ │ │ "content": [ -│ │ │ { -│ │ │ "text": "SKILL_DONE", -│ │ │ "type": "text" -│ │ │ } -│ │ │ ], -│ │ │ "isError": false, -│ │ │ "role": "toolResult", -│ │ │ "toolCallId": "", -│ │ │ "toolName": "bash" -│ │ │ } -│ │ │ ] -│ │ │ output: { -│ │ │ "content": [ -│ │ │ { -│ │ │ "text": "SKILL_DONE", -│ │ │ "textSignature": "{\"v\":1,\"id\":\"msg_054a16a2f2ea3217016a705b70a41481a38ece40ecddf69512\",\"phase\":\"final_answer\"}", -│ │ │ "type": "text" -│ │ │ } -│ │ │ ], -│ │ │ "role": "assistant" -│ │ │ } -│ │ │ metadata: { -│ │ │ "flue.api": "openai-responses", -│ │ │ "flue.input_message_offset": 17, -│ │ │ "flue.input_mode": "delta", -│ │ │ "flue.model": "gpt-5.4-nano", -│ │ │ "flue.provider": "openai", -│ │ │ "flue.session": "task:skill:", -│ │ │ "flue.stop_reason": "stop", -│ │ │ "flue.turn_purpose": "agent", -│ │ │ "model": "gpt-5.4-nano", -│ │ │ "provider": "openai" -│ │ │ } -│ │ │ metrics: { -│ │ │ "completion_tokens": 7, -│ │ │ "duration_ms": 0, -│ │ │ "estimated_cost": 0.00031035000000000004, -│ │ │ "prompt_cache_creation_tokens": 0, -│ │ │ "prompt_cached_tokens": 2560, -│ │ │ "prompt_tokens": 1252, -│ │ │ "tokens": 3819 -│ │ │ } -│ │ ├── tool:task [tool] -│ │ │ input: { -│ │ │ "cwd": ".", -│ │ │ "description": "Run the deterministic Flue instrumentation e2e test skill e2e-flue-skill.", -│ │ │ "prompt": "Run the skill named \"e2e-flue-skill\" with arguments {\"marker\":\"SKILL_DONE\"}. Output only the marker and no extra text when finished." -│ │ │ } -│ │ │ output: { -│ │ │ "content": [ -│ │ │ { -│ │ │ "text": "SKILL_DONE", -│ │ │ "type": "text" -│ │ │ } -│ │ │ ], -│ │ │ "details": { -│ │ │ "cwd": ".", -│ │ │ "messageId": "", -│ │ │ "session": "task:skill:", -│ │ │ "taskId": "" -│ │ │ } -│ │ │ } -│ │ │ metadata: { -│ │ │ "flue.session": "skill", -│ │ │ "flue.tool_name": "task", -│ │ │ "provider": "flue" -│ │ │ } -│ │ │ metrics: { -│ │ │ "duration_ms": 0 -│ │ │ } -│ │ └── flue.turn [llm] -│ │ input: [ -│ │ { -│ │ "content": [ -│ │ { -│ │ "text": "Run the skill named \"e2e-flue-skill\".\n\nArguments:\n{\n \"marker\": \"SKILL_DONE\"\n}", -│ │ "type": "text" -│ │ } -│ │ ], -│ │ "role": "user" -│ │ }, -│ │ { -│ │ "content": [ -│ │ { -│ │ "arguments": { -│ │ "cwd": ".", -│ │ "description": "Run the deterministic Flue instrumentation e2e test skill e2e-flue-skill.", -│ │ "prompt": "Run the skill named \"e2e-flue-skill\" with arguments {\"marker\":\"SKILL_DONE\"}. Output only the marker and no extra text when finished." -│ │ }, -│ │ "id": "", -│ │ "name": "task", -│ │ "type": "toolCall" -│ │ } -│ │ ], -│ │ "role": "assistant" -│ │ }, -│ │ { -│ │ "content": [ -│ │ { -│ │ "text": "SKILL_DONE", -│ │ "type": "text" -│ │ } -│ │ ], -│ │ "isError": false, -│ │ "role": "toolResult", -│ │ "toolCallId": "", -│ │ "toolName": "task" -│ │ } -│ │ ] -│ │ output: { -│ │ "content": [ -│ │ { -│ │ "text": "SKILL_DONE", -│ │ "textSignature": "{\"v\":1,\"id\":\"msg_0f099b3fce0721fd016a705b71558881a38c53a767e09d9e31\",\"phase\":\"final_answer\"}", -│ │ "type": "text" -│ │ } -│ │ ], -│ │ "role": "assistant" -│ │ } -│ │ metadata: { -│ │ "flue.api": "openai-responses", -│ │ "flue.model": "gpt-5.4-nano", -│ │ "flue.provider": "openai", -│ │ "flue.session": "skill", -│ │ "flue.stop_reason": "stop", -│ │ "flue.turn_purpose": "agent", -│ │ "model": "gpt-5.4-nano", -│ │ "provider": "openai" -│ │ } -│ │ metrics: { -│ │ "completion_tokens": 7, -│ │ "duration_ms": 0, -│ │ "estimated_cost": 0.00020255000000000002, -│ │ "prompt_cache_creation_tokens": 0, -│ │ "prompt_cached_tokens": 0, -│ │ "prompt_tokens": 969, -│ │ "tokens": 976 -│ │ } -│ ├── flue.task [task] -│ │ input: "Reply with exactly TASK_DONE and no other text." -│ │ output: "TASK_DONE" -│ │ metadata: { -│ │ "flue.session": "task", -│ │ "provider": "flue" -│ │ } -│ │ metrics: { -│ │ "duration_ms": 0 -│ │ } -│ │ └── flue.turn [llm] -│ │ input: [ -│ │ { -│ │ "content": [ -│ │ { -│ │ "text": "Reply with exactly TASK_DONE and no other text.", -│ │ "type": "text" -│ │ } -│ │ ], -│ │ "role": "user" -│ │ } -│ │ ] -│ │ output: { -│ │ "content": [ -│ │ { -│ │ "text": "TASK_DONE", -│ │ "textSignature": "{\"v\":1,\"id\":\"msg_0f3adfdb1510c165016a705b72843c81a0b30d242da683a882\"}", -│ │ "type": "text" -│ │ } -│ │ ], -│ │ "role": "assistant" -│ │ } -│ │ metadata: { -│ │ "flue.api": "openai-responses", -│ │ "flue.input_mode": "full", -│ │ "flue.model": "gpt-4o-mini", -│ │ "flue.provider": "openai", -│ │ "flue.session": "task:task:", -│ │ "flue.stop_reason": "stop", -│ │ "flue.turn_purpose": "agent", -│ │ "model": "gpt-4o-mini", -│ │ "provider": "openai" -│ │ } -│ │ metrics: { -│ │ "completion_tokens": 4, -│ │ "duration_ms": 0, -│ │ "estimated_cost": 0.0001263, -│ │ "prompt_cache_creation_tokens": 0, -│ │ "prompt_cached_tokens": 0, -│ │ "prompt_tokens": 826, -│ │ "tokens": 830 -│ │ } -│ └── flue.compact [task] -│ input: { -│ "estimatedTokens": 1229, -│ "reason": "manual" -│ } -│ output: { -│ "completed": true -│ } -│ metadata: { -│ "flue.operation": "compact", -│ "flue.session": "main", -│ "flue.workflow_name": "instrumentation", -│ "provider": "flue", -│ "scenario": "flue-instrumentation" -│ } -│ metrics: { -│ "duration_ms": 0 -│ } -│ └── compaction:manual [task] -│ input: { -│ "estimatedTokens": 1229, -│ "reason": "manual" -│ } -│ output: { -│ "messagesAfter": 2, -│ "messagesBefore": 8 -│ } -│ metadata: { -│ "flue.compaction_reason": "manual", -│ "flue.session": "main", -│ "provider": "flue" -│ } -│ metrics: { -│ "duration_ms": 0, -│ "messages_after": 2, -│ "messages_before": 8 -│ } -│ └── flue.turn [llm] -│ input: [ -│ { -│ "content": [ -│ { -│ "text": "\n[User]: Complete this instrumented research flow. Call exactly one tool per turn and wait for each tool result before choosing the next tool. Step 1: call lookup with query \"flue instrumentation\". Step 2: use the lookup result id as lookupId and call web_search with query \"Braintrust Flue reasoning stream instrumentation\". Step 3: use the first web_search result url and call summarize_source. After summarize_source returns, reply with exactly PROMPT_DONE and no other text.\n\n[Assistant tool calls]: lookup(query=\"flue instrumentation\")\n\n[Tool result]: {\"id\":\"flue-session-2026\",\"query\":\"flue instrumentation\",\"topic\":\"session instrumentation\"}\n\n[Assistant tool calls]: web_search(lookupId=\"flue-session-2026\", query=\"Braintrust Flue reasoning stream instrumentation\")\n\n[Tool result]: {\"lookupId\":\"flue-session-2026\",\"query\":\"Braintrust Flue reasoning stream instrumentation\",\"results\":[{\"title\":\"Flue reasoning stream instrumentation\",\"url\":\"https://example.test/flue/reasoning-streams\"}]}\n\n[Assistant tool calls]: summarize_source(url=\"https://example.test/flue/reasoning-streams\")\n\n[Tool result]: {\"summary\":\"Flue emits reasoning, tool execution, and LLM turn events separately.\",\"url\":\"https://example.test/flue/reasoning-streams\"}\n\n\nThis is the PREFIX of a turn that was too large to keep. The SUFFIX (recent work) is retained.\n\nSummarize the prefix to provide context for the retained suffix:\n\n## Original Request\n[What did the user ask for in this turn?]\n\n## Early Progress\n- [Key decisions and work done in the prefix]\n\n## Context for Suffix\n- [Information needed to understand the retained recent work]\n\nBe concise. Focus on what's needed to understand the kept suffix.", -│ "type": "text" -│ } -│ ], -│ "role": "user" -│ } -│ ] -│ output: { -│ "content": [ -│ { -│ "text": "## Original Request\nThe user requested a step-by-step execution of an instrumented research flow involving tool calls to gather information on \"flue instrumentation\" and \"", -│ "textSignature": "{\"v\":1,\"id\":\"msg_04d28ba03840bb65016a705b7358a081a38e69464057840036\"}", -│ "type": "text" -│ } -│ ], -│ "role": "assistant" -│ } -│ metadata: { -│ "flue.api": "openai-responses", -│ "flue.model": "gpt-4o-mini", -│ "flue.provider": "openai", -│ "flue.session": "main", -│ "flue.stop_reason": "stop", -│ "flue.turn_purpose": "compaction_prefix", -│ "model": "gpt-4o-mini", -│ "provider": "openai" -│ } -│ metrics: { -│ "completion_tokens": 0, -│ "duration_ms": 0, -│ "estimated_cost": 0, -│ "prompt_cache_creation_tokens": 0, -│ "prompt_cached_tokens": 0, -│ "prompt_tokens": 0, -│ "tokens": 0 -│ } -└── flue.prompt [task] - input: [ - { - "content": [ - { - "text": "Complete this instrumented research flow. Call exactly one tool per turn and wait for each tool result before choosing the next tool. Step 1: call lookup with query \"flue instrumentation\". Step 2: use the lookup result id as lookupId and call web_search with query \"Braintrust Flue reasoning stream instrumentation\". Step 3: use the first web_search result url and call summarize_source. After summarize_source returns, reply with exactly PROMPT_DONE and no other text.", - "type": "text" - } - ], - "role": "user" - } - ] - output: "PROMPT_DONE" - metadata: { - "flue.operation": "prompt", - "flue.session": "main", - "flue.workflow_name": "instrumentation", - "provider": "flue", - "scenario": "flue-instrumentation" - } - metrics: { - "duration_ms": 0 - } - ├── flue.turn [llm] - │ input: [ - │ { - │ "content": [ - │ { - │ "text": "Complete this instrumented research flow. Call exactly one tool per turn and wait for each tool result before choosing the next tool. Step 1: call lookup with query \"flue instrumentation\". Step 2: use the lookup result id as lookupId and call web_search with query \"Braintrust Flue reasoning stream instrumentation\". Step 3: use the first web_search result url and call summarize_source. After summarize_source returns, reply with exactly PROMPT_DONE and no other text.", - │ "type": "text" - │ } - │ ], - │ "role": "user" - │ } - │ ] - │ output: { - │ "content": [ - │ { - │ "arguments": { - │ "query": "flue instrumentation" - │ }, - │ "id": "", - │ "name": "lookup", - │ "type": "toolCall" - │ } - │ ], - │ "role": "assistant" - │ } - │ metadata: { - │ "flue.api": "openai-responses", - │ "flue.input_mode": "full", - │ "flue.model": "gpt-5.4-nano", - │ "flue.provider": "openai", - │ "flue.session": "main", - │ "flue.stop_reason": "toolUse", - │ "flue.turn_purpose": "agent", - │ "model": "gpt-5.4-nano", - │ "provider": "openai" - │ } - │ metrics: { - │ "completion_tokens": 19, - │ "duration_ms": 0, - │ "estimated_cost": 0.00022475000000000001, - │ "prompt_cache_creation_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 1005, - │ "tokens": 1024 - │ } - ├── tool:lookup [tool] - │ input: { - │ "query": "flue instrumentation" - │ } - │ output: { - │ "content": [ - │ { - │ "text": "{\"id\":\"flue-session-2026\",\"query\":\"flue instrumentation\",\"topic\":\"session instrumentation\"}", - │ "type": "text" - │ } - │ ], - │ "details": { - │ "customTool": "lookup" - │ } - │ } - │ metadata: { - │ "flue.session": "main", - │ "flue.tool_name": "lookup", - │ "provider": "flue" - │ } - │ metrics: { - │ "duration_ms": 0 - │ } - ├── flue.turn [llm] - │ input: [ - │ { - │ "content": [ - │ { - │ "arguments": { - │ "query": "flue instrumentation" - │ }, - │ "id": "", - │ "name": "lookup", - │ "type": "toolCall" - │ } - │ ], - │ "role": "assistant" - │ }, - │ { - │ "content": [ - │ { - │ "text": "{\"id\":\"flue-session-2026\",\"query\":\"flue instrumentation\",\"topic\":\"session instrumentation\"}", - │ "type": "text" - │ } - │ ], - │ "isError": false, - │ "role": "toolResult", - │ "toolCallId": "", - │ "toolName": "lookup" - │ } - │ ] - │ output: { - │ "content": [ - │ { - │ "arguments": { - │ "lookupId": "flue-session-2026", - │ "query": "Braintrust Flue reasoning stream instrumentation" - │ }, - │ "id": "", - │ "name": "web_search", - │ "type": "toolCall" - │ } - │ ], - │ "role": "assistant" - │ } - │ metadata: { - │ "flue.api": "openai-responses", - │ "flue.input_message_offset": 1, - │ "flue.input_mode": "delta", - │ "flue.model": "gpt-5.4-nano", - │ "flue.provider": "openai", - │ "flue.session": "main", - │ "flue.stop_reason": "toolUse", - │ "flue.turn_purpose": "agent", - │ "model": "gpt-5.4-nano", - │ "provider": "openai" - │ } - │ metrics: { - │ "completion_tokens": 34, - │ "duration_ms": 0, - │ "estimated_cost": 0.00025350000000000004, - │ "prompt_cache_creation_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 1055, - │ "tokens": 1089 - │ } - ├── tool:web_search [tool] - │ input: { - │ "lookupId": "flue-session-2026", - │ "query": "Braintrust Flue reasoning stream instrumentation" - │ } - │ output: { - │ "content": [ - │ { - │ "text": "{\"lookupId\":\"flue-session-2026\",\"query\":\"Braintrust Flue reasoning stream instrumentation\",\"results\":[{\"title\":\"Flue reasoning stream instrumentation\",\"url\":\"https://example.test/flue/reasoning-streams\"}]}", - │ "type": "text" - │ } - │ ], - │ "details": { - │ "customTool": "web_search" - │ } - │ } - │ metadata: { - │ "flue.session": "main", - │ "flue.tool_name": "web_search", - │ "provider": "flue" - │ } - │ metrics: { - │ "duration_ms": 0 - │ } - ├── flue.turn [llm] - │ input: [ - │ { - │ "content": [ - │ { - │ "arguments": { - │ "lookupId": "flue-session-2026", - │ "query": "Braintrust Flue reasoning stream instrumentation" - │ }, - │ "id": "", - │ "name": "web_search", - │ "type": "toolCall" - │ } - │ ], - │ "role": "assistant" - │ }, - │ { - │ "content": [ - │ { - │ "text": "{\"lookupId\":\"flue-session-2026\",\"query\":\"Braintrust Flue reasoning stream instrumentation\",\"results\":[{\"title\":\"Flue reasoning stream instrumentation\",\"url\":\"https://example.test/flue/reasoning-streams\"}]}", - │ "type": "text" - │ } - │ ], - │ "isError": false, - │ "role": "toolResult", - │ "toolCallId": "", - │ "toolName": "web_search" - │ } - │ ] - │ output: { - │ "content": [ - │ { - │ "arguments": { - │ "url": "https://example.test/flue/reasoning-streams" - │ }, - │ "id": "", - │ "name": "summarize_source", - │ "type": "toolCall" - │ } - │ ], - │ "role": "assistant" - │ } - │ metadata: { - │ "flue.api": "openai-responses", - │ "flue.input_message_offset": 3, - │ "flue.input_mode": "delta", - │ "flue.model": "gpt-5.4-nano", - │ "flue.provider": "openai", - │ "flue.session": "main", - │ "flue.stop_reason": "toolUse", - │ "flue.turn_purpose": "agent", - │ "model": "gpt-5.4-nano", - │ "provider": "openai" - │ } - │ metrics: { - │ "completion_tokens": 30, - │ "duration_ms": 0, - │ "estimated_cost": 0.00026690000000000004, - │ "prompt_cache_creation_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 1147, - │ "tokens": 1177 - │ } - ├── tool:summarize_source [tool] - │ input: { - │ "url": "https://example.test/flue/reasoning-streams" - │ } - │ output: { - │ "content": [ - │ { - │ "text": "{\"summary\":\"Flue emits reasoning, tool execution, and LLM turn events separately.\",\"url\":\"https://example.test/flue/reasoning-streams\"}", - │ "type": "text" - │ } - │ ], - │ "details": { - │ "customTool": "summarize_source" - │ } - │ } - │ metadata: { - │ "flue.session": "main", - │ "flue.tool_name": "summarize_source", - │ "provider": "flue" - │ } - │ metrics: { - │ "duration_ms": 0 - │ } - └── flue.turn [llm] - input: [ - { - "content": [ - { - "arguments": { - "url": "https://example.test/flue/reasoning-streams" - }, - "id": "", - "name": "summarize_source", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "{\"summary\":\"Flue emits reasoning, tool execution, and LLM turn events separately.\",\"url\":\"https://example.test/flue/reasoning-streams\"}", - "type": "text" - } - ], - "isError": false, - "role": "toolResult", - "toolCallId": "", - "toolName": "summarize_source" - } - ] - output: { - "content": [ - { - "text": "PROMPT_DONE", - "textSignature": "{\"v\":1,\"id\":\"msg_045458052dc10379016a705b62dbd8819189b8287061debf12\",\"phase\":\"final_answer\"}", - "type": "text" - } - ], - "role": "assistant" - } - metadata: { - "flue.api": "openai-responses", - "flue.input_message_offset": 5, - "flue.input_mode": "delta", - "flue.model": "gpt-5.4-nano", - "flue.provider": "openai", - "flue.session": "main", - "flue.stop_reason": "stop", - "flue.turn_purpose": "agent", - "model": "gpt-5.4-nano", - "provider": "openai" - } - metrics: { - "completion_tokens": 7, - "duration_ms": 0, - "estimated_cost": 0.00025315000000000005, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 1222, - "tokens": 1229 - } diff --git a/e2e/scenarios/flue-instrumentation/__snapshots__/flue-v0-8-latest-cli.span-tree.json b/e2e/scenarios/flue-instrumentation/__snapshots__/flue-v0-8-latest-cli.span-tree.json deleted file mode 100644 index e26112b02..000000000 --- a/e2e/scenarios/flue-instrumentation/__snapshots__/flue-v0-8-latest-cli.span-tree.json +++ /dev/null @@ -1,1662 +0,0 @@ -{ - "span_tree": [ - { - "name": "workflow:instrumentation", - "type": "task", - "children": [ - { - "name": "flue.skill", - "type": "task", - "children": [ - { - "name": "flue.turn", - "type": "llm", - "children": [ - { - "name": "flue.task", - "type": "task", - "children": [ - { - "name": "flue.turn", - "type": "llm", - "children": [], - "input": [ - { - "content": [ - { - "text": "Run the skill named \"e2e-flue-skill\" with arguments {\"marker\":\"SKILL_DONE\"}. Output only the marker and no extra text when finished.", - "type": "text" - } - ], - "role": "user" - } - ], - "output": { - "content": [ - { - "arguments": { - "agent": "e2e-flue-skill", - "cwd": ".", - "description": "Run e2e-flue-skill with marker SKILL_DONE", - "prompt": "Run the skill named \"e2e-flue-skill\" with arguments {\"marker\":\"SKILL_DONE\"}. Ensure completion and report the final output marker only." - }, - "id": "", - "name": "task", - "type": "toolCall" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "openai-responses", - "flue.input_mode": "full", - "flue.model": "gpt-5.4-nano", - "flue.provider": "openai", - "flue.session": "task:skill:", - "flue.stop_reason": "toolUse", - "flue.turn_purpose": "agent", - "model": "gpt-5.4-nano", - "provider": "openai" - }, - "metrics": { - "completion_tokens": 113, - "duration_ms": 0, - "estimated_cost": 0.00031225000000000003, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 855, - "tokens": 968 - } - }, - { - "name": "tool:task", - "type": "tool", - "children": [], - "input": { - "agent": "e2e-flue-skill", - "cwd": ".", - "description": "Run e2e-flue-skill with marker SKILL_DONE", - "prompt": "Run the skill named \"e2e-flue-skill\" with arguments {\"marker\":\"SKILL_DONE\"}. Ensure completion and report the final output marker only." - }, - "output": { - "content": [ - { - "text": "[flue] Subagent \"e2e-flue-skill\" is not declared. Available: (none).", - "type": "text" - } - ], - "details": {} - }, - "metadata": { - "flue.session": "task:skill:", - "flue.tool_name": "task", - "provider": "flue" - }, - "metrics": { - "duration_ms": 0 - }, - "error": "{\"content\":[{\"type\":\"text\",\"text\":\"[flue] Subagent \\\"e2e-flue-skill\\\" is not declared. Available: (none).\"}],\"details\":{}}" - }, - { - "name": "flue.turn", - "type": "llm", - "children": [], - "input": [ - { - "content": [ - { - "arguments": { - "agent": "e2e-flue-skill", - "cwd": ".", - "description": "Run e2e-flue-skill with marker SKILL_DONE", - "prompt": "Run the skill named \"e2e-flue-skill\" with arguments {\"marker\":\"SKILL_DONE\"}. Ensure completion and report the final output marker only." - }, - "id": "", - "name": "task", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "[flue] Subagent \"e2e-flue-skill\" is not declared. Available: (none).", - "type": "text" - } - ], - "isError": true, - "role": "toolResult", - "toolCallId": "", - "toolName": "task" - } - ], - "output": { - "content": [ - { - "arguments": { - "command": "find . -maxdepth 4 -name AGENTS.md -print", - "timeout": 100000 - }, - "id": "", - "name": "bash", - "type": "toolCall" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "openai-responses", - "flue.input_message_offset": 1, - "flue.input_mode": "delta", - "flue.model": "gpt-5.4-nano", - "flue.provider": "openai", - "flue.session": "task:skill:", - "flue.stop_reason": "toolUse", - "flue.turn_purpose": "agent", - "model": "gpt-5.4-nano", - "provider": "openai" - }, - "metrics": { - "completion_tokens": 94, - "duration_ms": 0, - "estimated_cost": 0.00031790000000000003, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 1002, - "tokens": 1096 - } - }, - { - "name": "tool:bash", - "type": "tool", - "children": [], - "input": { - "command": "find . -maxdepth 4 -name AGENTS.md -print", - "timeout": 100000 - }, - "output": { - "content": [ - { - "text": "(no output)", - "type": "text" - } - ], - "details": { - "command": "find . -maxdepth 4 -name AGENTS.md -print", - "exitCode": 0 - } - }, - "metadata": { - "flue.session": "task:skill:", - "flue.tool_name": "bash", - "provider": "flue" - }, - "metrics": { - "duration_ms": 0 - } - }, - { - "name": "flue.turn", - "type": "llm", - "children": [], - "input": [ - { - "content": [ - { - "arguments": { - "command": "find . -maxdepth 4 -name AGENTS.md -print", - "timeout": 100000 - }, - "id": "", - "name": "bash", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "(no output)", - "type": "text" - } - ], - "isError": false, - "role": "toolResult", - "toolCallId": "", - "toolName": "bash" - } - ], - "output": { - "content": [ - { - "arguments": { - "command": "find . -maxdepth 4 -type f -name AGENTS.md -print -exec sed -n '1,200p' {} \\;", - "timeout": 100000 - }, - "id": "", - "name": "bash", - "type": "toolCall" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "openai-responses", - "flue.input_message_offset": 3, - "flue.input_mode": "delta", - "flue.model": "gpt-5.4-nano", - "flue.provider": "openai", - "flue.session": "task:skill:", - "flue.stop_reason": "toolUse", - "flue.turn_purpose": "agent", - "model": "gpt-5.4-nano", - "provider": "openai" - }, - "metrics": { - "completion_tokens": 87, - "duration_ms": 0, - "estimated_cost": 0.00034815000000000003, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 1197, - "tokens": 1284 - } - }, - { - "name": "tool:bash", - "type": "tool", - "children": [], - "input": { - "command": "find . -maxdepth 4 -type f -name AGENTS.md -print -exec sed -n '1,200p' {} \\;", - "timeout": 100000 - }, - "output": { - "content": [ - { - "text": "(no output)", - "type": "text" - } - ], - "details": { - "command": "find . -maxdepth 4 -type f -name AGENTS.md -print -exec sed -n '1,200p' {} \\;", - "exitCode": 0 - } - }, - "metadata": { - "flue.session": "task:skill:", - "flue.tool_name": "bash", - "provider": "flue" - }, - "metrics": { - "duration_ms": 0 - } - }, - { - "name": "flue.turn", - "type": "llm", - "children": [], - "input": [ - { - "content": [ - { - "arguments": { - "command": "find . -maxdepth 4 -type f -name AGENTS.md -print -exec sed -n '1,200p' {} \\;", - "timeout": 100000 - }, - "id": "", - "name": "bash", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "(no output)", - "type": "text" - } - ], - "isError": false, - "role": "toolResult", - "toolCallId": "", - "toolName": "bash" - } - ], - "output": { - "content": [ - { - "arguments": { - "command": "find . -maxdepth 6 -type f -name 'AGENTS.md' -print", - "timeout": 100000 - }, - "id": "", - "name": "bash", - "type": "toolCall" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "openai-responses", - "flue.input_message_offset": 5, - "flue.input_mode": "delta", - "flue.model": "gpt-5.4-nano", - "flue.provider": "openai", - "flue.session": "task:skill:", - "flue.stop_reason": "toolUse", - "flue.turn_purpose": "agent", - "model": "gpt-5.4-nano", - "provider": "openai" - }, - "metrics": { - "completion_tokens": 58, - "duration_ms": 0, - "estimated_cost": 0.0003321, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 1298, - "tokens": 1356 - } - }, - { - "name": "tool:bash", - "type": "tool", - "children": [], - "input": { - "command": "find . -maxdepth 6 -type f -name 'AGENTS.md' -print", - "timeout": 100000 - }, - "output": { - "content": [ - { - "text": "(no output)", - "type": "text" - } - ], - "details": { - "command": "find . -maxdepth 6 -type f -name 'AGENTS.md' -print", - "exitCode": 0 - } - }, - "metadata": { - "flue.session": "task:skill:", - "flue.tool_name": "bash", - "provider": "flue" - }, - "metrics": { - "duration_ms": 0 - } - }, - { - "name": "flue.turn", - "type": "llm", - "children": [], - "input": [ - { - "content": [ - { - "arguments": { - "command": "find . -maxdepth 6 -type f -name 'AGENTS.md' -print", - "timeout": 100000 - }, - "id": "", - "name": "bash", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "(no output)", - "type": "text" - } - ], - "isError": false, - "role": "toolResult", - "toolCallId": "", - "toolName": "bash" - } - ], - "output": { - "content": [ - { - "arguments": { - "command": "find . -maxdepth 5 -type f -name '*flue*skill*' -o -name '*skill*' | head", - "timeout": 100000 - }, - "id": "", - "name": "bash", - "type": "toolCall" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "openai-responses", - "flue.input_message_offset": 7, - "flue.input_mode": "delta", - "flue.model": "gpt-5.4-nano", - "flue.provider": "openai", - "flue.session": "task:skill:", - "flue.stop_reason": "toolUse", - "flue.turn_purpose": "agent", - "model": "gpt-5.4-nano", - "provider": "openai" - }, - "metrics": { - "completion_tokens": 68, - "duration_ms": 0, - "estimated_cost": 0.00035900000000000005, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 1370, - "tokens": 1438 - } - }, - { - "name": "tool:bash", - "type": "tool", - "children": [], - "input": { - "command": "find . -maxdepth 5 -type f -name '*flue*skill*' -o -name '*skill*' | head", - "timeout": 100000 - }, - "output": { - "content": [ - { - "text": "./.agents/skills\n./.agents/skills/e2e-flue-skill", - "type": "text" - } - ], - "details": { - "command": "find . -maxdepth 5 -type f -name '*flue*skill*' -o -name '*skill*' | head", - "exitCode": 0 - } - }, - "metadata": { - "flue.session": "task:skill:", - "flue.tool_name": "bash", - "provider": "flue" - }, - "metrics": { - "duration_ms": 0 - } - }, - { - "name": "flue.turn", - "type": "llm", - "children": [], - "input": [ - { - "content": [ - { - "arguments": { - "command": "find . -maxdepth 5 -type f -name '*flue*skill*' -o -name '*skill*' | head", - "timeout": 100000 - }, - "id": "", - "name": "bash", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "./.agents/skills\n./.agents/skills/e2e-flue-skill", - "type": "text" - } - ], - "isError": false, - "role": "toolResult", - "toolCallId": "", - "toolName": "bash" - } - ], - "output": { - "content": [ - { - "arguments": { - "command": "find ./.agents/skills/e2e-flue-skill -maxdepth 2 -type f -print", - "timeout": 100000 - }, - "id": "", - "name": "bash", - "type": "toolCall" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "openai-responses", - "flue.input_message_offset": 9, - "flue.input_mode": "delta", - "flue.model": "gpt-5.4-nano", - "flue.provider": "openai", - "flue.session": "task:skill:", - "flue.stop_reason": "toolUse", - "flue.turn_purpose": "agent", - "model": "gpt-5.4-nano", - "provider": "openai" - }, - "metrics": { - "completion_tokens": 109, - "duration_ms": 0, - "estimated_cost": 0.00043885000000000007, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 1513, - "tokens": 1622 - } - }, - { - "name": "tool:bash", - "type": "tool", - "children": [], - "input": { - "command": "find ./.agents/skills/e2e-flue-skill -maxdepth 2 -type f -print", - "timeout": 100000 - }, - "output": { - "content": [ - { - "text": "./.agents/skills/e2e-flue-skill/SKILL.md", - "type": "text" - } - ], - "details": { - "command": "find ./.agents/skills/e2e-flue-skill -maxdepth 2 -type f -print", - "exitCode": 0 - } - }, - "metadata": { - "flue.session": "task:skill:", - "flue.tool_name": "bash", - "provider": "flue" - }, - "metrics": { - "duration_ms": 0 - } - }, - { - "name": "flue.turn", - "type": "llm", - "children": [], - "input": [ - { - "content": [ - { - "arguments": { - "command": "find ./.agents/skills/e2e-flue-skill -maxdepth 2 -type f -print", - "timeout": 100000 - }, - "id": "", - "name": "bash", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "./.agents/skills/e2e-flue-skill/SKILL.md", - "type": "text" - } - ], - "isError": false, - "role": "toolResult", - "toolCallId": "", - "toolName": "bash" - } - ], - "output": { - "content": [ - { - "arguments": { - "limit": 200, - "offset": 1, - "path": "./.agents/skills/e2e-flue-skill/SKILL.md" - }, - "id": "", - "name": "read", - "type": "toolCall" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "openai-responses", - "flue.input_message_offset": 11, - "flue.input_mode": "delta", - "flue.model": "gpt-5.4-nano", - "flue.provider": "openai", - "flue.session": "task:skill:", - "flue.stop_reason": "toolUse", - "flue.turn_purpose": "agent", - "model": "gpt-5.4-nano", - "provider": "openai" - }, - "metrics": { - "completion_tokens": 64, - "duration_ms": 0, - "estimated_cost": 0.00015592, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 1536, - "prompt_tokens": 226, - "tokens": 1826 - } - }, - { - "name": "tool:read", - "type": "tool", - "children": [], - "input": { - "limit": 200, - "offset": 1, - "path": "./.agents/skills/e2e-flue-skill/SKILL.md" - }, - "output": { - "content": [ - { - "text": "---\nname: e2e-flue-skill\ndescription: Deterministic skill used by the Braintrust Flue instrumentation e2e test.\n---\n\nReturn the marker from the provided args exactly once. Output no other text.\n", - "type": "text" - } - ], - "details": { - "lines": 7, - "path": "./.agents/skills/e2e-flue-skill/SKILL.md" - } - }, - "metadata": { - "flue.session": "task:skill:", - "flue.tool_name": "read", - "provider": "flue" - }, - "metrics": { - "duration_ms": 0 - } - }, - { - "name": "flue.turn", - "type": "llm", - "children": [], - "input": [ - { - "content": [ - { - "arguments": { - "limit": 200, - "offset": 1, - "path": "./.agents/skills/e2e-flue-skill/SKILL.md" - }, - "id": "", - "name": "read", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "---\nname: e2e-flue-skill\ndescription: Deterministic skill used by the Braintrust Flue instrumentation e2e test.\n---\n\nReturn the marker from the provided args exactly once. Output no other text.\n", - "type": "text" - } - ], - "isError": false, - "role": "toolResult", - "toolCallId": "", - "toolName": "read" - } - ], - "output": { - "content": [ - { - "arguments": { - "command": "cat ./package.json", - "timeout": 100000 - }, - "id": "", - "name": "bash", - "type": "toolCall" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "openai-responses", - "flue.input_message_offset": 13, - "flue.input_mode": "delta", - "flue.model": "gpt-5.4-nano", - "flue.provider": "openai", - "flue.session": "task:skill:", - "flue.stop_reason": "toolUse", - "flue.turn_purpose": "agent", - "model": "gpt-5.4-nano", - "provider": "openai" - }, - "metrics": { - "completion_tokens": 90, - "duration_ms": 0, - "estimated_cost": 0.00021242000000000003, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 1536, - "prompt_tokens": 346, - "tokens": 1972 - } - }, - { - "name": "tool:bash", - "type": "tool", - "children": [], - "input": { - "command": "cat ./package.json", - "timeout": 100000 - }, - "output": { - "content": [ - { - "text": "{\n \"name\": \"flue-instrumentation-scenario\",\n \"private\": true,\n \"type\": \"module\",\n \"braintrustScenario\": {\n \"bump\": {\n \"dependencies\": {\n \"flue-cli-v0-8-latest\": {\n \"package\": \"@flue/cli\",\n \"range\": \">=0.8.0 <0.9.0\"\n },\n \"flue-cli-v1-latest\": {\n \"package\": \"@flue/cli\",\n \"range\": \">=1.0.0-beta.3 <2.0.0-0\",\n \"allowPrerelease\": true\n },\n \"flue-runtime-v0-8-latest\": {\n \"package\": \"@flue/runtime\",\n \"range\": \">=0.8.0 <0.9.0\"\n },\n \"flue-runtime-v1-latest\": {\n \"package\": \"@flue/runtime\",\n \"range\": \">=1.0.0-beta.3 <2.0.0-0\",\n \"allowPrerelease\": true\n },\n \"flue-runtime-v2-latest\": {\n \"package\": \"@flue/runtime\",\n \"range\": \">=2.0.0 <3.0.0\"\n }\n }\n }\n },\n \"dependencies\": {\n \"@flue/cli\": \"0.8.0\",\n \"@flue/runtime\": \"0.8.0\",\n \"flue-cli-v0-8-latest\": \"npm:@flue/cli@0.8.1\",\n \"flue-cli-v1\": \"npm:@flue/cli@1.0.0-beta.3\",\n \"flue-cli-v1-latest\": \"npm:@flue/cli@1.0.0-beta.9\",\n \"flue-runtime-v0-8-latest\": \"npm:@flue/runtime@0.8.1\",\n \"flue-runtime-v1\": \"npm:@flue/runtime@1.0.0-beta.3\",\n \"flue-runtime-v1-latest\": \"npm:@flue/runtime@1.0.0-beta.9\",\n \"flue-runtime-v2\": \"npm:@flue/runtime@2.0.0\",\n \"flue-runtime-v2-latest\": \"npm:@flue/runtime@2.0.1\",\n \"pi-ai-v2\": \"npm:@earendil-works/pi-ai@0.83.0\",\n \"valibot\": \"1.4.2\"\n },\n \"pnpm\": {\n \"overrides\": {\n \"@flue/runtime@0.8.0>@earendil-works/pi-agent-core\": \"0.75.4\",\n \"@flue/runtime@0.8.0>@earendil-works/pi-ai\": \"0.75.4\",\n \"@flue/runtime@0.8.1>@earendil-works/pi-agent-core\": \"0.75.4\",\n \"@flue/runtime@0.8.1>@earendil-works/pi-ai\": \"0.75.4\",\n \"@hono/node-server@>=2.0.0 <2.0.10\": \"^2.0.10\",\n \"fast-uri@<3.1.5\": \"^3.1.5\",\n \"hono@>=4.0.0 <4.12.27\": \"^4.12.27\",\n \"brace-expansion@>=3.0.0 <5.0.9\": \"^5.0.9\",\n \"postcss@<8.5.18\": \"^8.5.18\",\n \"valibot@<1.4.2\": \"^1.4.2\",\n \"ip-address@<10.2.2\": \"^10.2.2\",\n \"undici@<7.29.0\": \"^7.29.0\"\n }\n }\n}", - "type": "text" - } - ], - "details": { - "command": "cat ./package.json", - "exitCode": 0 - } - }, - "metadata": { - "flue.session": "task:skill:", - "flue.tool_name": "bash", - "provider": "flue" - }, - "metrics": { - "duration_ms": 0 - } - }, - { - "name": "flue.turn", - "type": "llm", - "children": [], - "input": [ - { - "content": [ - { - "arguments": { - "command": "cat ./package.json", - "timeout": 100000 - }, - "id": "", - "name": "bash", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "{\n \"name\": \"flue-instrumentation-scenario\",\n \"private\": true,\n \"type\": \"module\",\n \"braintrustScenario\": {\n \"bump\": {\n \"dependencies\": {\n \"flue-cli-v0-8-latest\": {\n \"package\": \"@flue/cli\",\n \"range\": \">=0.8.0 <0.9.0\"\n },\n \"flue-cli-v1-latest\": {\n \"package\": \"@flue/cli\",\n \"range\": \">=1.0.0-beta.3 <2.0.0-0\",\n \"allowPrerelease\": true\n },\n \"flue-runtime-v0-8-latest\": {\n \"package\": \"@flue/runtime\",\n \"range\": \">=0.8.0 <0.9.0\"\n },\n \"flue-runtime-v1-latest\": {\n \"package\": \"@flue/runtime\",\n \"range\": \">=1.0.0-beta.3 <2.0.0-0\",\n \"allowPrerelease\": true\n },\n \"flue-runtime-v2-latest\": {\n \"package\": \"@flue/runtime\",\n \"range\": \">=2.0.0 <3.0.0\"\n }\n }\n }\n },\n \"dependencies\": {\n \"@flue/cli\": \"0.8.0\",\n \"@flue/runtime\": \"0.8.0\",\n \"flue-cli-v0-8-latest\": \"npm:@flue/cli@0.8.1\",\n \"flue-cli-v1\": \"npm:@flue/cli@1.0.0-beta.3\",\n \"flue-cli-v1-latest\": \"npm:@flue/cli@1.0.0-beta.9\",\n \"flue-runtime-v0-8-latest\": \"npm:@flue/runtime@0.8.1\",\n \"flue-runtime-v1\": \"npm:@flue/runtime@1.0.0-beta.3\",\n \"flue-runtime-v1-latest\": \"npm:@flue/runtime@1.0.0-beta.9\",\n \"flue-runtime-v2\": \"npm:@flue/runtime@2.0.0\",\n \"flue-runtime-v2-latest\": \"npm:@flue/runtime@2.0.1\",\n \"pi-ai-v2\": \"npm:@earendil-works/pi-ai@0.83.0\",\n \"valibot\": \"1.4.2\"\n },\n \"pnpm\": {\n \"overrides\": {\n \"@flue/runtime@0.8.0>@earendil-works/pi-agent-core\": \"0.75.4\",\n \"@flue/runtime@0.8.0>@earendil-works/pi-ai\": \"0.75.4\",\n \"@flue/runtime@0.8.1>@earendil-works/pi-agent-core\": \"0.75.4\",\n \"@flue/runtime@0.8.1>@earendil-works/pi-ai\": \"0.75.4\",\n \"@hono/node-server@>=2.0.0 <2.0.10\": \"^2.0.10\",\n \"fast-uri@<3.1.5\": \"^3.1.5\",\n \"hono@>=4.0.0 <4.12.27\": \"^4.12.27\",\n \"brace-expansion@>=3.0.0 <5.0.9\": \"^5.0.9\",\n \"postcss@<8.5.18\": \"^8.5.18\",\n \"valibot@<1.4.2\": \"^1.4.2\",\n \"ip-address@<10.2.2\": \"^10.2.2\",\n \"undici@<7.29.0\": \"^7.29.0\"\n }\n }\n}", - "type": "text" - } - ], - "isError": false, - "role": "toolResult", - "toolCallId": "", - "toolName": "bash" - } - ], - "output": { - "content": [ - { - "arguments": { - "command": "printf SKILL_DONE", - "timeout": 100000 - }, - "id": "", - "name": "bash", - "type": "toolCall" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "openai-responses", - "flue.input_message_offset": 15, - "flue.input_mode": "delta", - "flue.model": "gpt-5.4-nano", - "flue.provider": "openai", - "flue.session": "task:skill:", - "flue.stop_reason": "toolUse", - "flue.turn_purpose": "agent", - "model": "gpt-5.4-nano", - "provider": "openai" - }, - "metrics": { - "completion_tokens": 59, - "duration_ms": 0, - "estimated_cost": 0.0004814700000000001, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 1536, - "prompt_tokens": 1885, - "tokens": 3480 - } - }, - { - "name": "tool:bash", - "type": "tool", - "children": [], - "input": { - "command": "printf SKILL_DONE", - "timeout": 100000 - }, - "output": { - "content": [ - { - "text": "SKILL_DONE", - "type": "text" - } - ], - "details": { - "command": "printf SKILL_DONE", - "exitCode": 0 - } - }, - "metadata": { - "flue.session": "task:skill:", - "flue.tool_name": "bash", - "provider": "flue" - }, - "metrics": { - "duration_ms": 0 - } - }, - { - "name": "flue.turn", - "type": "llm", - "children": [], - "input": [ - { - "content": [ - { - "arguments": { - "command": "printf SKILL_DONE", - "timeout": 100000 - }, - "id": "", - "name": "bash", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "SKILL_DONE", - "type": "text" - } - ], - "isError": false, - "role": "toolResult", - "toolCallId": "", - "toolName": "bash" - } - ], - "output": { - "content": [ - { - "text": "SKILL_DONE", - "textSignature": "{\"v\":1,\"id\":\"msg_054a16a2f2ea3217016a705b70a41481a38ece40ecddf69512\",\"phase\":\"final_answer\"}", - "type": "text" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "openai-responses", - "flue.input_message_offset": 17, - "flue.input_mode": "delta", - "flue.model": "gpt-5.4-nano", - "flue.provider": "openai", - "flue.session": "task:skill:", - "flue.stop_reason": "stop", - "flue.turn_purpose": "agent", - "model": "gpt-5.4-nano", - "provider": "openai" - }, - "metrics": { - "completion_tokens": 7, - "duration_ms": 0, - "estimated_cost": 0.00031035000000000004, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 2560, - "prompt_tokens": 1252, - "tokens": 3819 - } - } - ], - "input": "Run the skill named \"e2e-flue-skill\" with arguments {\"marker\":\"SKILL_DONE\"}. Output only the marker and no extra text when finished.", - "output": "SKILL_DONE", - "metadata": { - "flue.session": "skill", - "provider": "flue" - }, - "metrics": { - "duration_ms": 0 - } - } - ], - "input": [ - { - "content": [ - { - "text": "Run the skill named \"e2e-flue-skill\".\n\nArguments:\n{\n \"marker\": \"SKILL_DONE\"\n}", - "type": "text" - } - ], - "role": "user" - } - ], - "output": { - "content": [ - { - "arguments": { - "cwd": ".", - "description": "Run the deterministic Flue instrumentation e2e test skill e2e-flue-skill.", - "prompt": "Run the skill named \"e2e-flue-skill\" with arguments {\"marker\":\"SKILL_DONE\"}. Output only the marker and no extra text when finished." - }, - "id": "", - "name": "task", - "type": "toolCall" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "openai-responses", - "flue.model": "gpt-5.4-nano", - "flue.provider": "openai", - "flue.session": "skill", - "flue.stop_reason": "toolUse", - "flue.turn_purpose": "agent", - "model": "gpt-5.4-nano", - "provider": "openai" - }, - "metrics": { - "completion_tokens": 109, - "duration_ms": 0, - "estimated_cost": 0.00030565000000000003, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 847, - "tokens": 956 - } - }, - { - "name": "tool:task", - "type": "tool", - "children": [], - "input": { - "cwd": ".", - "description": "Run the deterministic Flue instrumentation e2e test skill e2e-flue-skill.", - "prompt": "Run the skill named \"e2e-flue-skill\" with arguments {\"marker\":\"SKILL_DONE\"}. Output only the marker and no extra text when finished." - }, - "output": { - "content": [ - { - "text": "SKILL_DONE", - "type": "text" - } - ], - "details": { - "cwd": ".", - "messageId": "", - "session": "task:skill:", - "taskId": "" - } - }, - "metadata": { - "flue.session": "skill", - "flue.tool_name": "task", - "provider": "flue" - }, - "metrics": { - "duration_ms": 0 - } - }, - { - "name": "flue.turn", - "type": "llm", - "children": [], - "input": [ - { - "content": [ - { - "text": "Run the skill named \"e2e-flue-skill\".\n\nArguments:\n{\n \"marker\": \"SKILL_DONE\"\n}", - "type": "text" - } - ], - "role": "user" - }, - { - "content": [ - { - "arguments": { - "cwd": ".", - "description": "Run the deterministic Flue instrumentation e2e test skill e2e-flue-skill.", - "prompt": "Run the skill named \"e2e-flue-skill\" with arguments {\"marker\":\"SKILL_DONE\"}. Output only the marker and no extra text when finished." - }, - "id": "", - "name": "task", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "SKILL_DONE", - "type": "text" - } - ], - "isError": false, - "role": "toolResult", - "toolCallId": "", - "toolName": "task" - } - ], - "output": { - "content": [ - { - "text": "SKILL_DONE", - "textSignature": "{\"v\":1,\"id\":\"msg_0f099b3fce0721fd016a705b71558881a38c53a767e09d9e31\",\"phase\":\"final_answer\"}", - "type": "text" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "openai-responses", - "flue.model": "gpt-5.4-nano", - "flue.provider": "openai", - "flue.session": "skill", - "flue.stop_reason": "stop", - "flue.turn_purpose": "agent", - "model": "gpt-5.4-nano", - "provider": "openai" - }, - "metrics": { - "completion_tokens": 7, - "duration_ms": 0, - "estimated_cost": 0.00020255000000000002, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 969, - "tokens": 976 - } - } - ], - "input": [ - { - "content": [ - { - "text": "Run the skill named \"e2e-flue-skill\".\n\nArguments:\n{\n \"marker\": \"SKILL_DONE\"\n}", - "type": "text" - } - ], - "role": "user" - } - ], - "output": "SKILL_DONE", - "metadata": { - "flue.operation": "skill", - "flue.session": "skill", - "flue.workflow_name": "instrumentation", - "provider": "flue", - "scenario": "flue-instrumentation" - }, - "metrics": { - "duration_ms": 0 - } - }, - { - "name": "flue.task", - "type": "task", - "children": [ - { - "name": "flue.turn", - "type": "llm", - "children": [], - "input": [ - { - "content": [ - { - "text": "Reply with exactly TASK_DONE and no other text.", - "type": "text" - } - ], - "role": "user" - } - ], - "output": { - "content": [ - { - "text": "TASK_DONE", - "textSignature": "{\"v\":1,\"id\":\"msg_0f3adfdb1510c165016a705b72843c81a0b30d242da683a882\"}", - "type": "text" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "openai-responses", - "flue.input_mode": "full", - "flue.model": "gpt-4o-mini", - "flue.provider": "openai", - "flue.session": "task:task:", - "flue.stop_reason": "stop", - "flue.turn_purpose": "agent", - "model": "gpt-4o-mini", - "provider": "openai" - }, - "metrics": { - "completion_tokens": 4, - "duration_ms": 0, - "estimated_cost": 0.0001263, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 826, - "tokens": 830 - } - } - ], - "input": "Reply with exactly TASK_DONE and no other text.", - "output": "TASK_DONE", - "metadata": { - "flue.session": "task", - "provider": "flue" - }, - "metrics": { - "duration_ms": 0 - } - }, - { - "name": "flue.compact", - "type": "task", - "children": [ - { - "name": "compaction:manual", - "type": "task", - "children": [ - { - "name": "flue.turn", - "type": "llm", - "children": [], - "input": [ - { - "content": [ - { - "text": "\n[User]: Complete this instrumented research flow. Call exactly one tool per turn and wait for each tool result before choosing the next tool. Step 1: call lookup with query \"flue instrumentation\". Step 2: use the lookup result id as lookupId and call web_search with query \"Braintrust Flue reasoning stream instrumentation\". Step 3: use the first web_search result url and call summarize_source. After summarize_source returns, reply with exactly PROMPT_DONE and no other text.\n\n[Assistant tool calls]: lookup(query=\"flue instrumentation\")\n\n[Tool result]: {\"id\":\"flue-session-2026\",\"query\":\"flue instrumentation\",\"topic\":\"session instrumentation\"}\n\n[Assistant tool calls]: web_search(lookupId=\"flue-session-2026\", query=\"Braintrust Flue reasoning stream instrumentation\")\n\n[Tool result]: {\"lookupId\":\"flue-session-2026\",\"query\":\"Braintrust Flue reasoning stream instrumentation\",\"results\":[{\"title\":\"Flue reasoning stream instrumentation\",\"url\":\"https://example.test/flue/reasoning-streams\"}]}\n\n[Assistant tool calls]: summarize_source(url=\"https://example.test/flue/reasoning-streams\")\n\n[Tool result]: {\"summary\":\"Flue emits reasoning, tool execution, and LLM turn events separately.\",\"url\":\"https://example.test/flue/reasoning-streams\"}\n\n\nThis is the PREFIX of a turn that was too large to keep. The SUFFIX (recent work) is retained.\n\nSummarize the prefix to provide context for the retained suffix:\n\n## Original Request\n[What did the user ask for in this turn?]\n\n## Early Progress\n- [Key decisions and work done in the prefix]\n\n## Context for Suffix\n- [Information needed to understand the retained recent work]\n\nBe concise. Focus on what's needed to understand the kept suffix.", - "type": "text" - } - ], - "role": "user" - } - ], - "output": { - "content": [ - { - "text": "## Original Request\nThe user requested a step-by-step execution of an instrumented research flow involving tool calls to gather information on \"flue instrumentation\" and \"", - "textSignature": "{\"v\":1,\"id\":\"msg_04d28ba03840bb65016a705b7358a081a38e69464057840036\"}", - "type": "text" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "openai-responses", - "flue.model": "gpt-4o-mini", - "flue.provider": "openai", - "flue.session": "main", - "flue.stop_reason": "stop", - "flue.turn_purpose": "compaction_prefix", - "model": "gpt-4o-mini", - "provider": "openai" - }, - "metrics": { - "completion_tokens": 0, - "duration_ms": 0, - "estimated_cost": 0, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 0, - "tokens": 0 - } - } - ], - "input": { - "estimatedTokens": 1229, - "reason": "manual" - }, - "output": { - "messagesAfter": 2, - "messagesBefore": 8 - }, - "metadata": { - "flue.compaction_reason": "manual", - "flue.session": "main", - "provider": "flue" - }, - "metrics": { - "duration_ms": 0, - "messages_after": 2, - "messages_before": 8 - } - } - ], - "input": { - "estimatedTokens": 1229, - "reason": "manual" - }, - "output": { - "completed": true - }, - "metadata": { - "flue.operation": "compact", - "flue.session": "main", - "flue.workflow_name": "instrumentation", - "provider": "flue", - "scenario": "flue-instrumentation" - }, - "metrics": { - "duration_ms": 0 - } - } - ], - "input": { - "metadata": { - "scenario": "flue-instrumentation", - "testRunId": "" - }, - "scenario": "flue-instrumentation" - }, - "output": { - "scenario": "flue-instrumentation", - "status": "done" - }, - "metadata": { - "flue.workflow_name": "instrumentation", - "provider": "flue", - "scenario": "flue-instrumentation" - }, - "metrics": { - "duration_ms": 0 - } - }, - { - "name": "flue.prompt", - "type": "task", - "children": [ - { - "name": "flue.turn", - "type": "llm", - "children": [], - "input": [ - { - "content": [ - { - "text": "Complete this instrumented research flow. Call exactly one tool per turn and wait for each tool result before choosing the next tool. Step 1: call lookup with query \"flue instrumentation\". Step 2: use the lookup result id as lookupId and call web_search with query \"Braintrust Flue reasoning stream instrumentation\". Step 3: use the first web_search result url and call summarize_source. After summarize_source returns, reply with exactly PROMPT_DONE and no other text.", - "type": "text" - } - ], - "role": "user" - } - ], - "output": { - "content": [ - { - "arguments": { - "query": "flue instrumentation" - }, - "id": "", - "name": "lookup", - "type": "toolCall" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "openai-responses", - "flue.input_mode": "full", - "flue.model": "gpt-5.4-nano", - "flue.provider": "openai", - "flue.session": "main", - "flue.stop_reason": "toolUse", - "flue.turn_purpose": "agent", - "model": "gpt-5.4-nano", - "provider": "openai" - }, - "metrics": { - "completion_tokens": 19, - "duration_ms": 0, - "estimated_cost": 0.00022475000000000001, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 1005, - "tokens": 1024 - } - }, - { - "name": "tool:lookup", - "type": "tool", - "children": [], - "input": { - "query": "flue instrumentation" - }, - "output": { - "content": [ - { - "text": "{\"id\":\"flue-session-2026\",\"query\":\"flue instrumentation\",\"topic\":\"session instrumentation\"}", - "type": "text" - } - ], - "details": { - "customTool": "lookup" - } - }, - "metadata": { - "flue.session": "main", - "flue.tool_name": "lookup", - "provider": "flue" - }, - "metrics": { - "duration_ms": 0 - } - }, - { - "name": "flue.turn", - "type": "llm", - "children": [], - "input": [ - { - "content": [ - { - "arguments": { - "query": "flue instrumentation" - }, - "id": "", - "name": "lookup", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "{\"id\":\"flue-session-2026\",\"query\":\"flue instrumentation\",\"topic\":\"session instrumentation\"}", - "type": "text" - } - ], - "isError": false, - "role": "toolResult", - "toolCallId": "", - "toolName": "lookup" - } - ], - "output": { - "content": [ - { - "arguments": { - "lookupId": "flue-session-2026", - "query": "Braintrust Flue reasoning stream instrumentation" - }, - "id": "", - "name": "web_search", - "type": "toolCall" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "openai-responses", - "flue.input_message_offset": 1, - "flue.input_mode": "delta", - "flue.model": "gpt-5.4-nano", - "flue.provider": "openai", - "flue.session": "main", - "flue.stop_reason": "toolUse", - "flue.turn_purpose": "agent", - "model": "gpt-5.4-nano", - "provider": "openai" - }, - "metrics": { - "completion_tokens": 34, - "duration_ms": 0, - "estimated_cost": 0.00025350000000000004, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 1055, - "tokens": 1089 - } - }, - { - "name": "tool:web_search", - "type": "tool", - "children": [], - "input": { - "lookupId": "flue-session-2026", - "query": "Braintrust Flue reasoning stream instrumentation" - }, - "output": { - "content": [ - { - "text": "{\"lookupId\":\"flue-session-2026\",\"query\":\"Braintrust Flue reasoning stream instrumentation\",\"results\":[{\"title\":\"Flue reasoning stream instrumentation\",\"url\":\"https://example.test/flue/reasoning-streams\"}]}", - "type": "text" - } - ], - "details": { - "customTool": "web_search" - } - }, - "metadata": { - "flue.session": "main", - "flue.tool_name": "web_search", - "provider": "flue" - }, - "metrics": { - "duration_ms": 0 - } - }, - { - "name": "flue.turn", - "type": "llm", - "children": [], - "input": [ - { - "content": [ - { - "arguments": { - "lookupId": "flue-session-2026", - "query": "Braintrust Flue reasoning stream instrumentation" - }, - "id": "", - "name": "web_search", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "{\"lookupId\":\"flue-session-2026\",\"query\":\"Braintrust Flue reasoning stream instrumentation\",\"results\":[{\"title\":\"Flue reasoning stream instrumentation\",\"url\":\"https://example.test/flue/reasoning-streams\"}]}", - "type": "text" - } - ], - "isError": false, - "role": "toolResult", - "toolCallId": "", - "toolName": "web_search" - } - ], - "output": { - "content": [ - { - "arguments": { - "url": "https://example.test/flue/reasoning-streams" - }, - "id": "", - "name": "summarize_source", - "type": "toolCall" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "openai-responses", - "flue.input_message_offset": 3, - "flue.input_mode": "delta", - "flue.model": "gpt-5.4-nano", - "flue.provider": "openai", - "flue.session": "main", - "flue.stop_reason": "toolUse", - "flue.turn_purpose": "agent", - "model": "gpt-5.4-nano", - "provider": "openai" - }, - "metrics": { - "completion_tokens": 30, - "duration_ms": 0, - "estimated_cost": 0.00026690000000000004, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 1147, - "tokens": 1177 - } - }, - { - "name": "tool:summarize_source", - "type": "tool", - "children": [], - "input": { - "url": "https://example.test/flue/reasoning-streams" - }, - "output": { - "content": [ - { - "text": "{\"summary\":\"Flue emits reasoning, tool execution, and LLM turn events separately.\",\"url\":\"https://example.test/flue/reasoning-streams\"}", - "type": "text" - } - ], - "details": { - "customTool": "summarize_source" - } - }, - "metadata": { - "flue.session": "main", - "flue.tool_name": "summarize_source", - "provider": "flue" - }, - "metrics": { - "duration_ms": 0 - } - }, - { - "name": "flue.turn", - "type": "llm", - "children": [], - "input": [ - { - "content": [ - { - "arguments": { - "url": "https://example.test/flue/reasoning-streams" - }, - "id": "", - "name": "summarize_source", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "{\"summary\":\"Flue emits reasoning, tool execution, and LLM turn events separately.\",\"url\":\"https://example.test/flue/reasoning-streams\"}", - "type": "text" - } - ], - "isError": false, - "role": "toolResult", - "toolCallId": "", - "toolName": "summarize_source" - } - ], - "output": { - "content": [ - { - "text": "PROMPT_DONE", - "textSignature": "{\"v\":1,\"id\":\"msg_045458052dc10379016a705b62dbd8819189b8287061debf12\",\"phase\":\"final_answer\"}", - "type": "text" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "openai-responses", - "flue.input_message_offset": 5, - "flue.input_mode": "delta", - "flue.model": "gpt-5.4-nano", - "flue.provider": "openai", - "flue.session": "main", - "flue.stop_reason": "stop", - "flue.turn_purpose": "agent", - "model": "gpt-5.4-nano", - "provider": "openai" - }, - "metrics": { - "completion_tokens": 7, - "duration_ms": 0, - "estimated_cost": 0.00025315000000000005, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 1222, - "tokens": 1229 - } - } - ], - "input": [ - { - "content": [ - { - "text": "Complete this instrumented research flow. Call exactly one tool per turn and wait for each tool result before choosing the next tool. Step 1: call lookup with query \"flue instrumentation\". Step 2: use the lookup result id as lookupId and call web_search with query \"Braintrust Flue reasoning stream instrumentation\". Step 3: use the first web_search result url and call summarize_source. After summarize_source returns, reply with exactly PROMPT_DONE and no other text.", - "type": "text" - } - ], - "role": "user" - } - ], - "output": "PROMPT_DONE", - "metadata": { - "flue.operation": "prompt", - "flue.session": "main", - "flue.workflow_name": "instrumentation", - "provider": "flue", - "scenario": "flue-instrumentation" - }, - "metrics": { - "duration_ms": 0 - } - } - ] -} diff --git a/e2e/scenarios/flue-instrumentation/__snapshots__/flue-v0-8-latest-cli.span-tree.txt b/e2e/scenarios/flue-instrumentation/__snapshots__/flue-v0-8-latest-cli.span-tree.txt deleted file mode 100644 index abf057e06..000000000 --- a/e2e/scenarios/flue-instrumentation/__snapshots__/flue-v0-8-latest-cli.span-tree.txt +++ /dev/null @@ -1,1499 +0,0 @@ -span_tree: -├── workflow:instrumentation [task] -│ input: { -│ "metadata": { -│ "scenario": "flue-instrumentation", -│ "testRunId": "" -│ }, -│ "scenario": "flue-instrumentation" -│ } -│ output: { -│ "scenario": "flue-instrumentation", -│ "status": "done" -│ } -│ metadata: { -│ "flue.workflow_name": "instrumentation", -│ "provider": "flue", -│ "scenario": "flue-instrumentation" -│ } -│ metrics: { -│ "duration_ms": 0 -│ } -│ ├── flue.skill [task] -│ │ input: [ -│ │ { -│ │ "content": [ -│ │ { -│ │ "text": "Run the skill named \"e2e-flue-skill\".\n\nArguments:\n{\n \"marker\": \"SKILL_DONE\"\n}", -│ │ "type": "text" -│ │ } -│ │ ], -│ │ "role": "user" -│ │ } -│ │ ] -│ │ output: "SKILL_DONE" -│ │ metadata: { -│ │ "flue.operation": "skill", -│ │ "flue.session": "skill", -│ │ "flue.workflow_name": "instrumentation", -│ │ "provider": "flue", -│ │ "scenario": "flue-instrumentation" -│ │ } -│ │ metrics: { -│ │ "duration_ms": 0 -│ │ } -│ │ ├── flue.turn [llm] -│ │ │ input: [ -│ │ │ { -│ │ │ "content": [ -│ │ │ { -│ │ │ "text": "Run the skill named \"e2e-flue-skill\".\n\nArguments:\n{\n \"marker\": \"SKILL_DONE\"\n}", -│ │ │ "type": "text" -│ │ │ } -│ │ │ ], -│ │ │ "role": "user" -│ │ │ } -│ │ │ ] -│ │ │ output: { -│ │ │ "content": [ -│ │ │ { -│ │ │ "arguments": { -│ │ │ "cwd": ".", -│ │ │ "description": "Run the deterministic Flue instrumentation e2e test skill e2e-flue-skill.", -│ │ │ "prompt": "Run the skill named \"e2e-flue-skill\" with arguments {\"marker\":\"SKILL_DONE\"}. Output only the marker and no extra text when finished." -│ │ │ }, -│ │ │ "id": "", -│ │ │ "name": "task", -│ │ │ "type": "toolCall" -│ │ │ } -│ │ │ ], -│ │ │ "role": "assistant" -│ │ │ } -│ │ │ metadata: { -│ │ │ "flue.api": "openai-responses", -│ │ │ "flue.model": "gpt-5.4-nano", -│ │ │ "flue.provider": "openai", -│ │ │ "flue.session": "skill", -│ │ │ "flue.stop_reason": "toolUse", -│ │ │ "flue.turn_purpose": "agent", -│ │ │ "model": "gpt-5.4-nano", -│ │ │ "provider": "openai" -│ │ │ } -│ │ │ metrics: { -│ │ │ "completion_tokens": 109, -│ │ │ "duration_ms": 0, -│ │ │ "estimated_cost": 0.00030565000000000003, -│ │ │ "prompt_cache_creation_tokens": 0, -│ │ │ "prompt_cached_tokens": 0, -│ │ │ "prompt_tokens": 847, -│ │ │ "tokens": 956 -│ │ │ } -│ │ │ └── flue.task [task] -│ │ │ input: "Run the skill named \"e2e-flue-skill\" with arguments {\"marker\":\"SKILL_DONE\"}. Output only the marker and no extra text when finished." -│ │ │ output: "SKILL_DONE" -│ │ │ metadata: { -│ │ │ "flue.session": "skill", -│ │ │ "provider": "flue" -│ │ │ } -│ │ │ metrics: { -│ │ │ "duration_ms": 0 -│ │ │ } -│ │ │ ├── flue.turn [llm] -│ │ │ │ input: [ -│ │ │ │ { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "text": "Run the skill named \"e2e-flue-skill\" with arguments {\"marker\":\"SKILL_DONE\"}. Output only the marker and no extra text when finished.", -│ │ │ │ "type": "text" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "role": "user" -│ │ │ │ } -│ │ │ │ ] -│ │ │ │ output: { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "arguments": { -│ │ │ │ "agent": "e2e-flue-skill", -│ │ │ │ "cwd": ".", -│ │ │ │ "description": "Run e2e-flue-skill with marker SKILL_DONE", -│ │ │ │ "prompt": "Run the skill named \"e2e-flue-skill\" with arguments {\"marker\":\"SKILL_DONE\"}. Ensure completion and report the final output marker only." -│ │ │ │ }, -│ │ │ │ "id": "", -│ │ │ │ "name": "task", -│ │ │ │ "type": "toolCall" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "role": "assistant" -│ │ │ │ } -│ │ │ │ metadata: { -│ │ │ │ "flue.api": "openai-responses", -│ │ │ │ "flue.input_mode": "full", -│ │ │ │ "flue.model": "gpt-5.4-nano", -│ │ │ │ "flue.provider": "openai", -│ │ │ │ "flue.session": "task:skill:", -│ │ │ │ "flue.stop_reason": "toolUse", -│ │ │ │ "flue.turn_purpose": "agent", -│ │ │ │ "model": "gpt-5.4-nano", -│ │ │ │ "provider": "openai" -│ │ │ │ } -│ │ │ │ metrics: { -│ │ │ │ "completion_tokens": 113, -│ │ │ │ "duration_ms": 0, -│ │ │ │ "estimated_cost": 0.00031225000000000003, -│ │ │ │ "prompt_cache_creation_tokens": 0, -│ │ │ │ "prompt_cached_tokens": 0, -│ │ │ │ "prompt_tokens": 855, -│ │ │ │ "tokens": 968 -│ │ │ │ } -│ │ │ ├── tool:task [tool] -│ │ │ │ input: { -│ │ │ │ "agent": "e2e-flue-skill", -│ │ │ │ "cwd": ".", -│ │ │ │ "description": "Run e2e-flue-skill with marker SKILL_DONE", -│ │ │ │ "prompt": "Run the skill named \"e2e-flue-skill\" with arguments {\"marker\":\"SKILL_DONE\"}. Ensure completion and report the final output marker only." -│ │ │ │ } -│ │ │ │ output: { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "text": "[flue] Subagent \"e2e-flue-skill\" is not declared. Available: (none).", -│ │ │ │ "type": "text" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "details": {} -│ │ │ │ } -│ │ │ │ metadata: { -│ │ │ │ "flue.session": "task:skill:", -│ │ │ │ "flue.tool_name": "task", -│ │ │ │ "provider": "flue" -│ │ │ │ } -│ │ │ │ metrics: { -│ │ │ │ "duration_ms": 0 -│ │ │ │ } -│ │ │ │ error: "{\"content\":[{\"type\":\"text\",\"text\":\"[flue] Subagent \\\"e2e-flue-skill\\\" is not declared. Available: (none).\"}],\"details\":{}}" -│ │ │ ├── flue.turn [llm] -│ │ │ │ input: [ -│ │ │ │ { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "arguments": { -│ │ │ │ "agent": "e2e-flue-skill", -│ │ │ │ "cwd": ".", -│ │ │ │ "description": "Run e2e-flue-skill with marker SKILL_DONE", -│ │ │ │ "prompt": "Run the skill named \"e2e-flue-skill\" with arguments {\"marker\":\"SKILL_DONE\"}. Ensure completion and report the final output marker only." -│ │ │ │ }, -│ │ │ │ "id": "", -│ │ │ │ "name": "task", -│ │ │ │ "type": "toolCall" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "role": "assistant" -│ │ │ │ }, -│ │ │ │ { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "text": "[flue] Subagent \"e2e-flue-skill\" is not declared. Available: (none).", -│ │ │ │ "type": "text" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "isError": true, -│ │ │ │ "role": "toolResult", -│ │ │ │ "toolCallId": "", -│ │ │ │ "toolName": "task" -│ │ │ │ } -│ │ │ │ ] -│ │ │ │ output: { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "arguments": { -│ │ │ │ "command": "find . -maxdepth 4 -name AGENTS.md -print", -│ │ │ │ "timeout": 100000 -│ │ │ │ }, -│ │ │ │ "id": "", -│ │ │ │ "name": "bash", -│ │ │ │ "type": "toolCall" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "role": "assistant" -│ │ │ │ } -│ │ │ │ metadata: { -│ │ │ │ "flue.api": "openai-responses", -│ │ │ │ "flue.input_message_offset": 1, -│ │ │ │ "flue.input_mode": "delta", -│ │ │ │ "flue.model": "gpt-5.4-nano", -│ │ │ │ "flue.provider": "openai", -│ │ │ │ "flue.session": "task:skill:", -│ │ │ │ "flue.stop_reason": "toolUse", -│ │ │ │ "flue.turn_purpose": "agent", -│ │ │ │ "model": "gpt-5.4-nano", -│ │ │ │ "provider": "openai" -│ │ │ │ } -│ │ │ │ metrics: { -│ │ │ │ "completion_tokens": 94, -│ │ │ │ "duration_ms": 0, -│ │ │ │ "estimated_cost": 0.00031790000000000003, -│ │ │ │ "prompt_cache_creation_tokens": 0, -│ │ │ │ "prompt_cached_tokens": 0, -│ │ │ │ "prompt_tokens": 1002, -│ │ │ │ "tokens": 1096 -│ │ │ │ } -│ │ │ ├── tool:bash [tool] -│ │ │ │ input: { -│ │ │ │ "command": "find . -maxdepth 4 -name AGENTS.md -print", -│ │ │ │ "timeout": 100000 -│ │ │ │ } -│ │ │ │ output: { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "text": "(no output)", -│ │ │ │ "type": "text" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "details": { -│ │ │ │ "command": "find . -maxdepth 4 -name AGENTS.md -print", -│ │ │ │ "exitCode": 0 -│ │ │ │ } -│ │ │ │ } -│ │ │ │ metadata: { -│ │ │ │ "flue.session": "task:skill:", -│ │ │ │ "flue.tool_name": "bash", -│ │ │ │ "provider": "flue" -│ │ │ │ } -│ │ │ │ metrics: { -│ │ │ │ "duration_ms": 0 -│ │ │ │ } -│ │ │ ├── flue.turn [llm] -│ │ │ │ input: [ -│ │ │ │ { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "arguments": { -│ │ │ │ "command": "find . -maxdepth 4 -name AGENTS.md -print", -│ │ │ │ "timeout": 100000 -│ │ │ │ }, -│ │ │ │ "id": "", -│ │ │ │ "name": "bash", -│ │ │ │ "type": "toolCall" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "role": "assistant" -│ │ │ │ }, -│ │ │ │ { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "text": "(no output)", -│ │ │ │ "type": "text" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "isError": false, -│ │ │ │ "role": "toolResult", -│ │ │ │ "toolCallId": "", -│ │ │ │ "toolName": "bash" -│ │ │ │ } -│ │ │ │ ] -│ │ │ │ output: { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "arguments": { -│ │ │ │ "command": "find . -maxdepth 4 -type f -name AGENTS.md -print -exec sed -n '1,200p' {} \\;", -│ │ │ │ "timeout": 100000 -│ │ │ │ }, -│ │ │ │ "id": "", -│ │ │ │ "name": "bash", -│ │ │ │ "type": "toolCall" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "role": "assistant" -│ │ │ │ } -│ │ │ │ metadata: { -│ │ │ │ "flue.api": "openai-responses", -│ │ │ │ "flue.input_message_offset": 3, -│ │ │ │ "flue.input_mode": "delta", -│ │ │ │ "flue.model": "gpt-5.4-nano", -│ │ │ │ "flue.provider": "openai", -│ │ │ │ "flue.session": "task:skill:", -│ │ │ │ "flue.stop_reason": "toolUse", -│ │ │ │ "flue.turn_purpose": "agent", -│ │ │ │ "model": "gpt-5.4-nano", -│ │ │ │ "provider": "openai" -│ │ │ │ } -│ │ │ │ metrics: { -│ │ │ │ "completion_tokens": 87, -│ │ │ │ "duration_ms": 0, -│ │ │ │ "estimated_cost": 0.00034815000000000003, -│ │ │ │ "prompt_cache_creation_tokens": 0, -│ │ │ │ "prompt_cached_tokens": 0, -│ │ │ │ "prompt_tokens": 1197, -│ │ │ │ "tokens": 1284 -│ │ │ │ } -│ │ │ ├── tool:bash [tool] -│ │ │ │ input: { -│ │ │ │ "command": "find . -maxdepth 4 -type f -name AGENTS.md -print -exec sed -n '1,200p' {} \\;", -│ │ │ │ "timeout": 100000 -│ │ │ │ } -│ │ │ │ output: { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "text": "(no output)", -│ │ │ │ "type": "text" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "details": { -│ │ │ │ "command": "find . -maxdepth 4 -type f -name AGENTS.md -print -exec sed -n '1,200p' {} \\;", -│ │ │ │ "exitCode": 0 -│ │ │ │ } -│ │ │ │ } -│ │ │ │ metadata: { -│ │ │ │ "flue.session": "task:skill:", -│ │ │ │ "flue.tool_name": "bash", -│ │ │ │ "provider": "flue" -│ │ │ │ } -│ │ │ │ metrics: { -│ │ │ │ "duration_ms": 0 -│ │ │ │ } -│ │ │ ├── flue.turn [llm] -│ │ │ │ input: [ -│ │ │ │ { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "arguments": { -│ │ │ │ "command": "find . -maxdepth 4 -type f -name AGENTS.md -print -exec sed -n '1,200p' {} \\;", -│ │ │ │ "timeout": 100000 -│ │ │ │ }, -│ │ │ │ "id": "", -│ │ │ │ "name": "bash", -│ │ │ │ "type": "toolCall" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "role": "assistant" -│ │ │ │ }, -│ │ │ │ { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "text": "(no output)", -│ │ │ │ "type": "text" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "isError": false, -│ │ │ │ "role": "toolResult", -│ │ │ │ "toolCallId": "", -│ │ │ │ "toolName": "bash" -│ │ │ │ } -│ │ │ │ ] -│ │ │ │ output: { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "arguments": { -│ │ │ │ "command": "find . -maxdepth 6 -type f -name 'AGENTS.md' -print", -│ │ │ │ "timeout": 100000 -│ │ │ │ }, -│ │ │ │ "id": "", -│ │ │ │ "name": "bash", -│ │ │ │ "type": "toolCall" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "role": "assistant" -│ │ │ │ } -│ │ │ │ metadata: { -│ │ │ │ "flue.api": "openai-responses", -│ │ │ │ "flue.input_message_offset": 5, -│ │ │ │ "flue.input_mode": "delta", -│ │ │ │ "flue.model": "gpt-5.4-nano", -│ │ │ │ "flue.provider": "openai", -│ │ │ │ "flue.session": "task:skill:", -│ │ │ │ "flue.stop_reason": "toolUse", -│ │ │ │ "flue.turn_purpose": "agent", -│ │ │ │ "model": "gpt-5.4-nano", -│ │ │ │ "provider": "openai" -│ │ │ │ } -│ │ │ │ metrics: { -│ │ │ │ "completion_tokens": 58, -│ │ │ │ "duration_ms": 0, -│ │ │ │ "estimated_cost": 0.0003321, -│ │ │ │ "prompt_cache_creation_tokens": 0, -│ │ │ │ "prompt_cached_tokens": 0, -│ │ │ │ "prompt_tokens": 1298, -│ │ │ │ "tokens": 1356 -│ │ │ │ } -│ │ │ ├── tool:bash [tool] -│ │ │ │ input: { -│ │ │ │ "command": "find . -maxdepth 6 -type f -name 'AGENTS.md' -print", -│ │ │ │ "timeout": 100000 -│ │ │ │ } -│ │ │ │ output: { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "text": "(no output)", -│ │ │ │ "type": "text" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "details": { -│ │ │ │ "command": "find . -maxdepth 6 -type f -name 'AGENTS.md' -print", -│ │ │ │ "exitCode": 0 -│ │ │ │ } -│ │ │ │ } -│ │ │ │ metadata: { -│ │ │ │ "flue.session": "task:skill:", -│ │ │ │ "flue.tool_name": "bash", -│ │ │ │ "provider": "flue" -│ │ │ │ } -│ │ │ │ metrics: { -│ │ │ │ "duration_ms": 0 -│ │ │ │ } -│ │ │ ├── flue.turn [llm] -│ │ │ │ input: [ -│ │ │ │ { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "arguments": { -│ │ │ │ "command": "find . -maxdepth 6 -type f -name 'AGENTS.md' -print", -│ │ │ │ "timeout": 100000 -│ │ │ │ }, -│ │ │ │ "id": "", -│ │ │ │ "name": "bash", -│ │ │ │ "type": "toolCall" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "role": "assistant" -│ │ │ │ }, -│ │ │ │ { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "text": "(no output)", -│ │ │ │ "type": "text" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "isError": false, -│ │ │ │ "role": "toolResult", -│ │ │ │ "toolCallId": "", -│ │ │ │ "toolName": "bash" -│ │ │ │ } -│ │ │ │ ] -│ │ │ │ output: { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "arguments": { -│ │ │ │ "command": "find . -maxdepth 5 -type f -name '*flue*skill*' -o -name '*skill*' | head", -│ │ │ │ "timeout": 100000 -│ │ │ │ }, -│ │ │ │ "id": "", -│ │ │ │ "name": "bash", -│ │ │ │ "type": "toolCall" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "role": "assistant" -│ │ │ │ } -│ │ │ │ metadata: { -│ │ │ │ "flue.api": "openai-responses", -│ │ │ │ "flue.input_message_offset": 7, -│ │ │ │ "flue.input_mode": "delta", -│ │ │ │ "flue.model": "gpt-5.4-nano", -│ │ │ │ "flue.provider": "openai", -│ │ │ │ "flue.session": "task:skill:", -│ │ │ │ "flue.stop_reason": "toolUse", -│ │ │ │ "flue.turn_purpose": "agent", -│ │ │ │ "model": "gpt-5.4-nano", -│ │ │ │ "provider": "openai" -│ │ │ │ } -│ │ │ │ metrics: { -│ │ │ │ "completion_tokens": 68, -│ │ │ │ "duration_ms": 0, -│ │ │ │ "estimated_cost": 0.00035900000000000005, -│ │ │ │ "prompt_cache_creation_tokens": 0, -│ │ │ │ "prompt_cached_tokens": 0, -│ │ │ │ "prompt_tokens": 1370, -│ │ │ │ "tokens": 1438 -│ │ │ │ } -│ │ │ ├── tool:bash [tool] -│ │ │ │ input: { -│ │ │ │ "command": "find . -maxdepth 5 -type f -name '*flue*skill*' -o -name '*skill*' | head", -│ │ │ │ "timeout": 100000 -│ │ │ │ } -│ │ │ │ output: { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "text": "./.agents/skills\n./.agents/skills/e2e-flue-skill", -│ │ │ │ "type": "text" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "details": { -│ │ │ │ "command": "find . -maxdepth 5 -type f -name '*flue*skill*' -o -name '*skill*' | head", -│ │ │ │ "exitCode": 0 -│ │ │ │ } -│ │ │ │ } -│ │ │ │ metadata: { -│ │ │ │ "flue.session": "task:skill:", -│ │ │ │ "flue.tool_name": "bash", -│ │ │ │ "provider": "flue" -│ │ │ │ } -│ │ │ │ metrics: { -│ │ │ │ "duration_ms": 0 -│ │ │ │ } -│ │ │ ├── flue.turn [llm] -│ │ │ │ input: [ -│ │ │ │ { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "arguments": { -│ │ │ │ "command": "find . -maxdepth 5 -type f -name '*flue*skill*' -o -name '*skill*' | head", -│ │ │ │ "timeout": 100000 -│ │ │ │ }, -│ │ │ │ "id": "", -│ │ │ │ "name": "bash", -│ │ │ │ "type": "toolCall" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "role": "assistant" -│ │ │ │ }, -│ │ │ │ { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "text": "./.agents/skills\n./.agents/skills/e2e-flue-skill", -│ │ │ │ "type": "text" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "isError": false, -│ │ │ │ "role": "toolResult", -│ │ │ │ "toolCallId": "", -│ │ │ │ "toolName": "bash" -│ │ │ │ } -│ │ │ │ ] -│ │ │ │ output: { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "arguments": { -│ │ │ │ "command": "find ./.agents/skills/e2e-flue-skill -maxdepth 2 -type f -print", -│ │ │ │ "timeout": 100000 -│ │ │ │ }, -│ │ │ │ "id": "", -│ │ │ │ "name": "bash", -│ │ │ │ "type": "toolCall" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "role": "assistant" -│ │ │ │ } -│ │ │ │ metadata: { -│ │ │ │ "flue.api": "openai-responses", -│ │ │ │ "flue.input_message_offset": 9, -│ │ │ │ "flue.input_mode": "delta", -│ │ │ │ "flue.model": "gpt-5.4-nano", -│ │ │ │ "flue.provider": "openai", -│ │ │ │ "flue.session": "task:skill:", -│ │ │ │ "flue.stop_reason": "toolUse", -│ │ │ │ "flue.turn_purpose": "agent", -│ │ │ │ "model": "gpt-5.4-nano", -│ │ │ │ "provider": "openai" -│ │ │ │ } -│ │ │ │ metrics: { -│ │ │ │ "completion_tokens": 109, -│ │ │ │ "duration_ms": 0, -│ │ │ │ "estimated_cost": 0.00043885000000000007, -│ │ │ │ "prompt_cache_creation_tokens": 0, -│ │ │ │ "prompt_cached_tokens": 0, -│ │ │ │ "prompt_tokens": 1513, -│ │ │ │ "tokens": 1622 -│ │ │ │ } -│ │ │ ├── tool:bash [tool] -│ │ │ │ input: { -│ │ │ │ "command": "find ./.agents/skills/e2e-flue-skill -maxdepth 2 -type f -print", -│ │ │ │ "timeout": 100000 -│ │ │ │ } -│ │ │ │ output: { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "text": "./.agents/skills/e2e-flue-skill/SKILL.md", -│ │ │ │ "type": "text" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "details": { -│ │ │ │ "command": "find ./.agents/skills/e2e-flue-skill -maxdepth 2 -type f -print", -│ │ │ │ "exitCode": 0 -│ │ │ │ } -│ │ │ │ } -│ │ │ │ metadata: { -│ │ │ │ "flue.session": "task:skill:", -│ │ │ │ "flue.tool_name": "bash", -│ │ │ │ "provider": "flue" -│ │ │ │ } -│ │ │ │ metrics: { -│ │ │ │ "duration_ms": 0 -│ │ │ │ } -│ │ │ ├── flue.turn [llm] -│ │ │ │ input: [ -│ │ │ │ { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "arguments": { -│ │ │ │ "command": "find ./.agents/skills/e2e-flue-skill -maxdepth 2 -type f -print", -│ │ │ │ "timeout": 100000 -│ │ │ │ }, -│ │ │ │ "id": "", -│ │ │ │ "name": "bash", -│ │ │ │ "type": "toolCall" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "role": "assistant" -│ │ │ │ }, -│ │ │ │ { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "text": "./.agents/skills/e2e-flue-skill/SKILL.md", -│ │ │ │ "type": "text" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "isError": false, -│ │ │ │ "role": "toolResult", -│ │ │ │ "toolCallId": "", -│ │ │ │ "toolName": "bash" -│ │ │ │ } -│ │ │ │ ] -│ │ │ │ output: { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "arguments": { -│ │ │ │ "limit": 200, -│ │ │ │ "offset": 1, -│ │ │ │ "path": "./.agents/skills/e2e-flue-skill/SKILL.md" -│ │ │ │ }, -│ │ │ │ "id": "", -│ │ │ │ "name": "read", -│ │ │ │ "type": "toolCall" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "role": "assistant" -│ │ │ │ } -│ │ │ │ metadata: { -│ │ │ │ "flue.api": "openai-responses", -│ │ │ │ "flue.input_message_offset": 11, -│ │ │ │ "flue.input_mode": "delta", -│ │ │ │ "flue.model": "gpt-5.4-nano", -│ │ │ │ "flue.provider": "openai", -│ │ │ │ "flue.session": "task:skill:", -│ │ │ │ "flue.stop_reason": "toolUse", -│ │ │ │ "flue.turn_purpose": "agent", -│ │ │ │ "model": "gpt-5.4-nano", -│ │ │ │ "provider": "openai" -│ │ │ │ } -│ │ │ │ metrics: { -│ │ │ │ "completion_tokens": 64, -│ │ │ │ "duration_ms": 0, -│ │ │ │ "estimated_cost": 0.00015592, -│ │ │ │ "prompt_cache_creation_tokens": 0, -│ │ │ │ "prompt_cached_tokens": 1536, -│ │ │ │ "prompt_tokens": 226, -│ │ │ │ "tokens": 1826 -│ │ │ │ } -│ │ │ ├── tool:read [tool] -│ │ │ │ input: { -│ │ │ │ "limit": 200, -│ │ │ │ "offset": 1, -│ │ │ │ "path": "./.agents/skills/e2e-flue-skill/SKILL.md" -│ │ │ │ } -│ │ │ │ output: { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "text": "---\nname: e2e-flue-skill\ndescription: Deterministic skill used by the Braintrust Flue instrumentation e2e test.\n---\n\nReturn the marker from the provided args exactly once. Output no other text.\n", -│ │ │ │ "type": "text" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "details": { -│ │ │ │ "lines": 7, -│ │ │ │ "path": "./.agents/skills/e2e-flue-skill/SKILL.md" -│ │ │ │ } -│ │ │ │ } -│ │ │ │ metadata: { -│ │ │ │ "flue.session": "task:skill:", -│ │ │ │ "flue.tool_name": "read", -│ │ │ │ "provider": "flue" -│ │ │ │ } -│ │ │ │ metrics: { -│ │ │ │ "duration_ms": 0 -│ │ │ │ } -│ │ │ ├── flue.turn [llm] -│ │ │ │ input: [ -│ │ │ │ { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "arguments": { -│ │ │ │ "limit": 200, -│ │ │ │ "offset": 1, -│ │ │ │ "path": "./.agents/skills/e2e-flue-skill/SKILL.md" -│ │ │ │ }, -│ │ │ │ "id": "", -│ │ │ │ "name": "read", -│ │ │ │ "type": "toolCall" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "role": "assistant" -│ │ │ │ }, -│ │ │ │ { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "text": "---\nname: e2e-flue-skill\ndescription: Deterministic skill used by the Braintrust Flue instrumentation e2e test.\n---\n\nReturn the marker from the provided args exactly once. Output no other text.\n", -│ │ │ │ "type": "text" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "isError": false, -│ │ │ │ "role": "toolResult", -│ │ │ │ "toolCallId": "", -│ │ │ │ "toolName": "read" -│ │ │ │ } -│ │ │ │ ] -│ │ │ │ output: { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "arguments": { -│ │ │ │ "command": "cat ./package.json", -│ │ │ │ "timeout": 100000 -│ │ │ │ }, -│ │ │ │ "id": "", -│ │ │ │ "name": "bash", -│ │ │ │ "type": "toolCall" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "role": "assistant" -│ │ │ │ } -│ │ │ │ metadata: { -│ │ │ │ "flue.api": "openai-responses", -│ │ │ │ "flue.input_message_offset": 13, -│ │ │ │ "flue.input_mode": "delta", -│ │ │ │ "flue.model": "gpt-5.4-nano", -│ │ │ │ "flue.provider": "openai", -│ │ │ │ "flue.session": "task:skill:", -│ │ │ │ "flue.stop_reason": "toolUse", -│ │ │ │ "flue.turn_purpose": "agent", -│ │ │ │ "model": "gpt-5.4-nano", -│ │ │ │ "provider": "openai" -│ │ │ │ } -│ │ │ │ metrics: { -│ │ │ │ "completion_tokens": 90, -│ │ │ │ "duration_ms": 0, -│ │ │ │ "estimated_cost": 0.00021242000000000003, -│ │ │ │ "prompt_cache_creation_tokens": 0, -│ │ │ │ "prompt_cached_tokens": 1536, -│ │ │ │ "prompt_tokens": 346, -│ │ │ │ "tokens": 1972 -│ │ │ │ } -│ │ │ ├── tool:bash [tool] -│ │ │ │ input: { -│ │ │ │ "command": "cat ./package.json", -│ │ │ │ "timeout": 100000 -│ │ │ │ } -│ │ │ │ output: { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "text": "{\n \"name\": \"flue-instrumentation-scenario\",\n \"private\": true,\n \"type\": \"module\",\n \"braintrustScenario\": {\n \"bump\": {\n \"dependencies\": {\n \"flue-cli-v0-8-latest\": {\n \"package\": \"@flue/cli\",\n \"range\": \">=0.8.0 <0.9.0\"\n },\n \"flue-cli-v1-latest\": {\n \"package\": \"@flue/cli\",\n \"range\": \">=1.0.0-beta.3 <2.0.0-0\",\n \"allowPrerelease\": true\n },\n \"flue-runtime-v0-8-latest\": {\n \"package\": \"@flue/runtime\",\n \"range\": \">=0.8.0 <0.9.0\"\n },\n \"flue-runtime-v1-latest\": {\n \"package\": \"@flue/runtime\",\n \"range\": \">=1.0.0-beta.3 <2.0.0-0\",\n \"allowPrerelease\": true\n },\n \"flue-runtime-v2-latest\": {\n \"package\": \"@flue/runtime\",\n \"range\": \">=2.0.0 <3.0.0\"\n }\n }\n }\n },\n \"dependencies\": {\n \"@flue/cli\": \"0.8.0\",\n \"@flue/runtime\": \"0.8.0\",\n \"flue-cli-v0-8-latest\": \"npm:@flue/cli@0.8.1\",\n \"flue-cli-v1\": \"npm:@flue/cli@1.0.0-beta.3\",\n \"flue-cli-v1-latest\": \"npm:@flue/cli@1.0.0-beta.9\",\n \"flue-runtime-v0-8-latest\": \"npm:@flue/runtime@0.8.1\",\n \"flue-runtime-v1\": \"npm:@flue/runtime@1.0.0-beta.3\",\n \"flue-runtime-v1-latest\": \"npm:@flue/runtime@1.0.0-beta.9\",\n \"flue-runtime-v2\": \"npm:@flue/runtime@2.0.0\",\n \"flue-runtime-v2-latest\": \"npm:@flue/runtime@2.0.1\",\n \"pi-ai-v2\": \"npm:@earendil-works/pi-ai@0.83.0\",\n \"valibot\": \"1.4.2\"\n },\n \"pnpm\": {\n \"overrides\": {\n \"@flue/runtime@0.8.0>@earendil-works/pi-agent-core\": \"0.75.4\",\n \"@flue/runtime@0.8.0>@earendil-works/pi-ai\": \"0.75.4\",\n \"@flue/runtime@0.8.1>@earendil-works/pi-agent-core\": \"0.75.4\",\n \"@flue/runtime@0.8.1>@earendil-works/pi-ai\": \"0.75.4\",\n \"@hono/node-server@>=2.0.0 <2.0.10\": \"^2.0.10\",\n \"fast-uri@<3.1.5\": \"^3.1.5\",\n \"hono@>=4.0.0 <4.12.27\": \"^4.12.27\",\n \"brace-expansion@>=3.0.0 <5.0.9\": \"^5.0.9\",\n \"postcss@<8.5.18\": \"^8.5.18\",\n \"valibot@<1.4.2\": \"^1.4.2\",\n \"ip-address@<10.2.2\": \"^10.2.2\",\n \"undici@<7.29.0\": \"^7.29.0\"\n }\n }\n}", -│ │ │ │ "type": "text" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "details": { -│ │ │ │ "command": "cat ./package.json", -│ │ │ │ "exitCode": 0 -│ │ │ │ } -│ │ │ │ } -│ │ │ │ metadata: { -│ │ │ │ "flue.session": "task:skill:", -│ │ │ │ "flue.tool_name": "bash", -│ │ │ │ "provider": "flue" -│ │ │ │ } -│ │ │ │ metrics: { -│ │ │ │ "duration_ms": 0 -│ │ │ │ } -│ │ │ ├── flue.turn [llm] -│ │ │ │ input: [ -│ │ │ │ { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "arguments": { -│ │ │ │ "command": "cat ./package.json", -│ │ │ │ "timeout": 100000 -│ │ │ │ }, -│ │ │ │ "id": "", -│ │ │ │ "name": "bash", -│ │ │ │ "type": "toolCall" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "role": "assistant" -│ │ │ │ }, -│ │ │ │ { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "text": "{\n \"name\": \"flue-instrumentation-scenario\",\n \"private\": true,\n \"type\": \"module\",\n \"braintrustScenario\": {\n \"bump\": {\n \"dependencies\": {\n \"flue-cli-v0-8-latest\": {\n \"package\": \"@flue/cli\",\n \"range\": \">=0.8.0 <0.9.0\"\n },\n \"flue-cli-v1-latest\": {\n \"package\": \"@flue/cli\",\n \"range\": \">=1.0.0-beta.3 <2.0.0-0\",\n \"allowPrerelease\": true\n },\n \"flue-runtime-v0-8-latest\": {\n \"package\": \"@flue/runtime\",\n \"range\": \">=0.8.0 <0.9.0\"\n },\n \"flue-runtime-v1-latest\": {\n \"package\": \"@flue/runtime\",\n \"range\": \">=1.0.0-beta.3 <2.0.0-0\",\n \"allowPrerelease\": true\n },\n \"flue-runtime-v2-latest\": {\n \"package\": \"@flue/runtime\",\n \"range\": \">=2.0.0 <3.0.0\"\n }\n }\n }\n },\n \"dependencies\": {\n \"@flue/cli\": \"0.8.0\",\n \"@flue/runtime\": \"0.8.0\",\n \"flue-cli-v0-8-latest\": \"npm:@flue/cli@0.8.1\",\n \"flue-cli-v1\": \"npm:@flue/cli@1.0.0-beta.3\",\n \"flue-cli-v1-latest\": \"npm:@flue/cli@1.0.0-beta.9\",\n \"flue-runtime-v0-8-latest\": \"npm:@flue/runtime@0.8.1\",\n \"flue-runtime-v1\": \"npm:@flue/runtime@1.0.0-beta.3\",\n \"flue-runtime-v1-latest\": \"npm:@flue/runtime@1.0.0-beta.9\",\n \"flue-runtime-v2\": \"npm:@flue/runtime@2.0.0\",\n \"flue-runtime-v2-latest\": \"npm:@flue/runtime@2.0.1\",\n \"pi-ai-v2\": \"npm:@earendil-works/pi-ai@0.83.0\",\n \"valibot\": \"1.4.2\"\n },\n \"pnpm\": {\n \"overrides\": {\n \"@flue/runtime@0.8.0>@earendil-works/pi-agent-core\": \"0.75.4\",\n \"@flue/runtime@0.8.0>@earendil-works/pi-ai\": \"0.75.4\",\n \"@flue/runtime@0.8.1>@earendil-works/pi-agent-core\": \"0.75.4\",\n \"@flue/runtime@0.8.1>@earendil-works/pi-ai\": \"0.75.4\",\n \"@hono/node-server@>=2.0.0 <2.0.10\": \"^2.0.10\",\n \"fast-uri@<3.1.5\": \"^3.1.5\",\n \"hono@>=4.0.0 <4.12.27\": \"^4.12.27\",\n \"brace-expansion@>=3.0.0 <5.0.9\": \"^5.0.9\",\n \"postcss@<8.5.18\": \"^8.5.18\",\n \"valibot@<1.4.2\": \"^1.4.2\",\n \"ip-address@<10.2.2\": \"^10.2.2\",\n \"undici@<7.29.0\": \"^7.29.0\"\n }\n }\n}", -│ │ │ │ "type": "text" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "isError": false, -│ │ │ │ "role": "toolResult", -│ │ │ │ "toolCallId": "", -│ │ │ │ "toolName": "bash" -│ │ │ │ } -│ │ │ │ ] -│ │ │ │ output: { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "arguments": { -│ │ │ │ "command": "printf SKILL_DONE", -│ │ │ │ "timeout": 100000 -│ │ │ │ }, -│ │ │ │ "id": "", -│ │ │ │ "name": "bash", -│ │ │ │ "type": "toolCall" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "role": "assistant" -│ │ │ │ } -│ │ │ │ metadata: { -│ │ │ │ "flue.api": "openai-responses", -│ │ │ │ "flue.input_message_offset": 15, -│ │ │ │ "flue.input_mode": "delta", -│ │ │ │ "flue.model": "gpt-5.4-nano", -│ │ │ │ "flue.provider": "openai", -│ │ │ │ "flue.session": "task:skill:", -│ │ │ │ "flue.stop_reason": "toolUse", -│ │ │ │ "flue.turn_purpose": "agent", -│ │ │ │ "model": "gpt-5.4-nano", -│ │ │ │ "provider": "openai" -│ │ │ │ } -│ │ │ │ metrics: { -│ │ │ │ "completion_tokens": 59, -│ │ │ │ "duration_ms": 0, -│ │ │ │ "estimated_cost": 0.0004814700000000001, -│ │ │ │ "prompt_cache_creation_tokens": 0, -│ │ │ │ "prompt_cached_tokens": 1536, -│ │ │ │ "prompt_tokens": 1885, -│ │ │ │ "tokens": 3480 -│ │ │ │ } -│ │ │ ├── tool:bash [tool] -│ │ │ │ input: { -│ │ │ │ "command": "printf SKILL_DONE", -│ │ │ │ "timeout": 100000 -│ │ │ │ } -│ │ │ │ output: { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "text": "SKILL_DONE", -│ │ │ │ "type": "text" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "details": { -│ │ │ │ "command": "printf SKILL_DONE", -│ │ │ │ "exitCode": 0 -│ │ │ │ } -│ │ │ │ } -│ │ │ │ metadata: { -│ │ │ │ "flue.session": "task:skill:", -│ │ │ │ "flue.tool_name": "bash", -│ │ │ │ "provider": "flue" -│ │ │ │ } -│ │ │ │ metrics: { -│ │ │ │ "duration_ms": 0 -│ │ │ │ } -│ │ │ └── flue.turn [llm] -│ │ │ input: [ -│ │ │ { -│ │ │ "content": [ -│ │ │ { -│ │ │ "arguments": { -│ │ │ "command": "printf SKILL_DONE", -│ │ │ "timeout": 100000 -│ │ │ }, -│ │ │ "id": "", -│ │ │ "name": "bash", -│ │ │ "type": "toolCall" -│ │ │ } -│ │ │ ], -│ │ │ "role": "assistant" -│ │ │ }, -│ │ │ { -│ │ │ "content": [ -│ │ │ { -│ │ │ "text": "SKILL_DONE", -│ │ │ "type": "text" -│ │ │ } -│ │ │ ], -│ │ │ "isError": false, -│ │ │ "role": "toolResult", -│ │ │ "toolCallId": "", -│ │ │ "toolName": "bash" -│ │ │ } -│ │ │ ] -│ │ │ output: { -│ │ │ "content": [ -│ │ │ { -│ │ │ "text": "SKILL_DONE", -│ │ │ "textSignature": "{\"v\":1,\"id\":\"msg_054a16a2f2ea3217016a705b70a41481a38ece40ecddf69512\",\"phase\":\"final_answer\"}", -│ │ │ "type": "text" -│ │ │ } -│ │ │ ], -│ │ │ "role": "assistant" -│ │ │ } -│ │ │ metadata: { -│ │ │ "flue.api": "openai-responses", -│ │ │ "flue.input_message_offset": 17, -│ │ │ "flue.input_mode": "delta", -│ │ │ "flue.model": "gpt-5.4-nano", -│ │ │ "flue.provider": "openai", -│ │ │ "flue.session": "task:skill:", -│ │ │ "flue.stop_reason": "stop", -│ │ │ "flue.turn_purpose": "agent", -│ │ │ "model": "gpt-5.4-nano", -│ │ │ "provider": "openai" -│ │ │ } -│ │ │ metrics: { -│ │ │ "completion_tokens": 7, -│ │ │ "duration_ms": 0, -│ │ │ "estimated_cost": 0.00031035000000000004, -│ │ │ "prompt_cache_creation_tokens": 0, -│ │ │ "prompt_cached_tokens": 2560, -│ │ │ "prompt_tokens": 1252, -│ │ │ "tokens": 3819 -│ │ │ } -│ │ ├── tool:task [tool] -│ │ │ input: { -│ │ │ "cwd": ".", -│ │ │ "description": "Run the deterministic Flue instrumentation e2e test skill e2e-flue-skill.", -│ │ │ "prompt": "Run the skill named \"e2e-flue-skill\" with arguments {\"marker\":\"SKILL_DONE\"}. Output only the marker and no extra text when finished." -│ │ │ } -│ │ │ output: { -│ │ │ "content": [ -│ │ │ { -│ │ │ "text": "SKILL_DONE", -│ │ │ "type": "text" -│ │ │ } -│ │ │ ], -│ │ │ "details": { -│ │ │ "cwd": ".", -│ │ │ "messageId": "", -│ │ │ "session": "task:skill:", -│ │ │ "taskId": "" -│ │ │ } -│ │ │ } -│ │ │ metadata: { -│ │ │ "flue.session": "skill", -│ │ │ "flue.tool_name": "task", -│ │ │ "provider": "flue" -│ │ │ } -│ │ │ metrics: { -│ │ │ "duration_ms": 0 -│ │ │ } -│ │ └── flue.turn [llm] -│ │ input: [ -│ │ { -│ │ "content": [ -│ │ { -│ │ "text": "Run the skill named \"e2e-flue-skill\".\n\nArguments:\n{\n \"marker\": \"SKILL_DONE\"\n}", -│ │ "type": "text" -│ │ } -│ │ ], -│ │ "role": "user" -│ │ }, -│ │ { -│ │ "content": [ -│ │ { -│ │ "arguments": { -│ │ "cwd": ".", -│ │ "description": "Run the deterministic Flue instrumentation e2e test skill e2e-flue-skill.", -│ │ "prompt": "Run the skill named \"e2e-flue-skill\" with arguments {\"marker\":\"SKILL_DONE\"}. Output only the marker and no extra text when finished." -│ │ }, -│ │ "id": "", -│ │ "name": "task", -│ │ "type": "toolCall" -│ │ } -│ │ ], -│ │ "role": "assistant" -│ │ }, -│ │ { -│ │ "content": [ -│ │ { -│ │ "text": "SKILL_DONE", -│ │ "type": "text" -│ │ } -│ │ ], -│ │ "isError": false, -│ │ "role": "toolResult", -│ │ "toolCallId": "", -│ │ "toolName": "task" -│ │ } -│ │ ] -│ │ output: { -│ │ "content": [ -│ │ { -│ │ "text": "SKILL_DONE", -│ │ "textSignature": "{\"v\":1,\"id\":\"msg_0f099b3fce0721fd016a705b71558881a38c53a767e09d9e31\",\"phase\":\"final_answer\"}", -│ │ "type": "text" -│ │ } -│ │ ], -│ │ "role": "assistant" -│ │ } -│ │ metadata: { -│ │ "flue.api": "openai-responses", -│ │ "flue.model": "gpt-5.4-nano", -│ │ "flue.provider": "openai", -│ │ "flue.session": "skill", -│ │ "flue.stop_reason": "stop", -│ │ "flue.turn_purpose": "agent", -│ │ "model": "gpt-5.4-nano", -│ │ "provider": "openai" -│ │ } -│ │ metrics: { -│ │ "completion_tokens": 7, -│ │ "duration_ms": 0, -│ │ "estimated_cost": 0.00020255000000000002, -│ │ "prompt_cache_creation_tokens": 0, -│ │ "prompt_cached_tokens": 0, -│ │ "prompt_tokens": 969, -│ │ "tokens": 976 -│ │ } -│ ├── flue.task [task] -│ │ input: "Reply with exactly TASK_DONE and no other text." -│ │ output: "TASK_DONE" -│ │ metadata: { -│ │ "flue.session": "task", -│ │ "provider": "flue" -│ │ } -│ │ metrics: { -│ │ "duration_ms": 0 -│ │ } -│ │ └── flue.turn [llm] -│ │ input: [ -│ │ { -│ │ "content": [ -│ │ { -│ │ "text": "Reply with exactly TASK_DONE and no other text.", -│ │ "type": "text" -│ │ } -│ │ ], -│ │ "role": "user" -│ │ } -│ │ ] -│ │ output: { -│ │ "content": [ -│ │ { -│ │ "text": "TASK_DONE", -│ │ "textSignature": "{\"v\":1,\"id\":\"msg_0f3adfdb1510c165016a705b72843c81a0b30d242da683a882\"}", -│ │ "type": "text" -│ │ } -│ │ ], -│ │ "role": "assistant" -│ │ } -│ │ metadata: { -│ │ "flue.api": "openai-responses", -│ │ "flue.input_mode": "full", -│ │ "flue.model": "gpt-4o-mini", -│ │ "flue.provider": "openai", -│ │ "flue.session": "task:task:", -│ │ "flue.stop_reason": "stop", -│ │ "flue.turn_purpose": "agent", -│ │ "model": "gpt-4o-mini", -│ │ "provider": "openai" -│ │ } -│ │ metrics: { -│ │ "completion_tokens": 4, -│ │ "duration_ms": 0, -│ │ "estimated_cost": 0.0001263, -│ │ "prompt_cache_creation_tokens": 0, -│ │ "prompt_cached_tokens": 0, -│ │ "prompt_tokens": 826, -│ │ "tokens": 830 -│ │ } -│ └── flue.compact [task] -│ input: { -│ "estimatedTokens": 1229, -│ "reason": "manual" -│ } -│ output: { -│ "completed": true -│ } -│ metadata: { -│ "flue.operation": "compact", -│ "flue.session": "main", -│ "flue.workflow_name": "instrumentation", -│ "provider": "flue", -│ "scenario": "flue-instrumentation" -│ } -│ metrics: { -│ "duration_ms": 0 -│ } -│ └── compaction:manual [task] -│ input: { -│ "estimatedTokens": 1229, -│ "reason": "manual" -│ } -│ output: { -│ "messagesAfter": 2, -│ "messagesBefore": 8 -│ } -│ metadata: { -│ "flue.compaction_reason": "manual", -│ "flue.session": "main", -│ "provider": "flue" -│ } -│ metrics: { -│ "duration_ms": 0, -│ "messages_after": 2, -│ "messages_before": 8 -│ } -│ └── flue.turn [llm] -│ input: [ -│ { -│ "content": [ -│ { -│ "text": "\n[User]: Complete this instrumented research flow. Call exactly one tool per turn and wait for each tool result before choosing the next tool. Step 1: call lookup with query \"flue instrumentation\". Step 2: use the lookup result id as lookupId and call web_search with query \"Braintrust Flue reasoning stream instrumentation\". Step 3: use the first web_search result url and call summarize_source. After summarize_source returns, reply with exactly PROMPT_DONE and no other text.\n\n[Assistant tool calls]: lookup(query=\"flue instrumentation\")\n\n[Tool result]: {\"id\":\"flue-session-2026\",\"query\":\"flue instrumentation\",\"topic\":\"session instrumentation\"}\n\n[Assistant tool calls]: web_search(lookupId=\"flue-session-2026\", query=\"Braintrust Flue reasoning stream instrumentation\")\n\n[Tool result]: {\"lookupId\":\"flue-session-2026\",\"query\":\"Braintrust Flue reasoning stream instrumentation\",\"results\":[{\"title\":\"Flue reasoning stream instrumentation\",\"url\":\"https://example.test/flue/reasoning-streams\"}]}\n\n[Assistant tool calls]: summarize_source(url=\"https://example.test/flue/reasoning-streams\")\n\n[Tool result]: {\"summary\":\"Flue emits reasoning, tool execution, and LLM turn events separately.\",\"url\":\"https://example.test/flue/reasoning-streams\"}\n\n\nThis is the PREFIX of a turn that was too large to keep. The SUFFIX (recent work) is retained.\n\nSummarize the prefix to provide context for the retained suffix:\n\n## Original Request\n[What did the user ask for in this turn?]\n\n## Early Progress\n- [Key decisions and work done in the prefix]\n\n## Context for Suffix\n- [Information needed to understand the retained recent work]\n\nBe concise. Focus on what's needed to understand the kept suffix.", -│ "type": "text" -│ } -│ ], -│ "role": "user" -│ } -│ ] -│ output: { -│ "content": [ -│ { -│ "text": "## Original Request\nThe user requested a step-by-step execution of an instrumented research flow involving tool calls to gather information on \"flue instrumentation\" and \"", -│ "textSignature": "{\"v\":1,\"id\":\"msg_04d28ba03840bb65016a705b7358a081a38e69464057840036\"}", -│ "type": "text" -│ } -│ ], -│ "role": "assistant" -│ } -│ metadata: { -│ "flue.api": "openai-responses", -│ "flue.model": "gpt-4o-mini", -│ "flue.provider": "openai", -│ "flue.session": "main", -│ "flue.stop_reason": "stop", -│ "flue.turn_purpose": "compaction_prefix", -│ "model": "gpt-4o-mini", -│ "provider": "openai" -│ } -│ metrics: { -│ "completion_tokens": 0, -│ "duration_ms": 0, -│ "estimated_cost": 0, -│ "prompt_cache_creation_tokens": 0, -│ "prompt_cached_tokens": 0, -│ "prompt_tokens": 0, -│ "tokens": 0 -│ } -└── flue.prompt [task] - input: [ - { - "content": [ - { - "text": "Complete this instrumented research flow. Call exactly one tool per turn and wait for each tool result before choosing the next tool. Step 1: call lookup with query \"flue instrumentation\". Step 2: use the lookup result id as lookupId and call web_search with query \"Braintrust Flue reasoning stream instrumentation\". Step 3: use the first web_search result url and call summarize_source. After summarize_source returns, reply with exactly PROMPT_DONE and no other text.", - "type": "text" - } - ], - "role": "user" - } - ] - output: "PROMPT_DONE" - metadata: { - "flue.operation": "prompt", - "flue.session": "main", - "flue.workflow_name": "instrumentation", - "provider": "flue", - "scenario": "flue-instrumentation" - } - metrics: { - "duration_ms": 0 - } - ├── flue.turn [llm] - │ input: [ - │ { - │ "content": [ - │ { - │ "text": "Complete this instrumented research flow. Call exactly one tool per turn and wait for each tool result before choosing the next tool. Step 1: call lookup with query \"flue instrumentation\". Step 2: use the lookup result id as lookupId and call web_search with query \"Braintrust Flue reasoning stream instrumentation\". Step 3: use the first web_search result url and call summarize_source. After summarize_source returns, reply with exactly PROMPT_DONE and no other text.", - │ "type": "text" - │ } - │ ], - │ "role": "user" - │ } - │ ] - │ output: { - │ "content": [ - │ { - │ "arguments": { - │ "query": "flue instrumentation" - │ }, - │ "id": "", - │ "name": "lookup", - │ "type": "toolCall" - │ } - │ ], - │ "role": "assistant" - │ } - │ metadata: { - │ "flue.api": "openai-responses", - │ "flue.input_mode": "full", - │ "flue.model": "gpt-5.4-nano", - │ "flue.provider": "openai", - │ "flue.session": "main", - │ "flue.stop_reason": "toolUse", - │ "flue.turn_purpose": "agent", - │ "model": "gpt-5.4-nano", - │ "provider": "openai" - │ } - │ metrics: { - │ "completion_tokens": 19, - │ "duration_ms": 0, - │ "estimated_cost": 0.00022475000000000001, - │ "prompt_cache_creation_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 1005, - │ "tokens": 1024 - │ } - ├── tool:lookup [tool] - │ input: { - │ "query": "flue instrumentation" - │ } - │ output: { - │ "content": [ - │ { - │ "text": "{\"id\":\"flue-session-2026\",\"query\":\"flue instrumentation\",\"topic\":\"session instrumentation\"}", - │ "type": "text" - │ } - │ ], - │ "details": { - │ "customTool": "lookup" - │ } - │ } - │ metadata: { - │ "flue.session": "main", - │ "flue.tool_name": "lookup", - │ "provider": "flue" - │ } - │ metrics: { - │ "duration_ms": 0 - │ } - ├── flue.turn [llm] - │ input: [ - │ { - │ "content": [ - │ { - │ "arguments": { - │ "query": "flue instrumentation" - │ }, - │ "id": "", - │ "name": "lookup", - │ "type": "toolCall" - │ } - │ ], - │ "role": "assistant" - │ }, - │ { - │ "content": [ - │ { - │ "text": "{\"id\":\"flue-session-2026\",\"query\":\"flue instrumentation\",\"topic\":\"session instrumentation\"}", - │ "type": "text" - │ } - │ ], - │ "isError": false, - │ "role": "toolResult", - │ "toolCallId": "", - │ "toolName": "lookup" - │ } - │ ] - │ output: { - │ "content": [ - │ { - │ "arguments": { - │ "lookupId": "flue-session-2026", - │ "query": "Braintrust Flue reasoning stream instrumentation" - │ }, - │ "id": "", - │ "name": "web_search", - │ "type": "toolCall" - │ } - │ ], - │ "role": "assistant" - │ } - │ metadata: { - │ "flue.api": "openai-responses", - │ "flue.input_message_offset": 1, - │ "flue.input_mode": "delta", - │ "flue.model": "gpt-5.4-nano", - │ "flue.provider": "openai", - │ "flue.session": "main", - │ "flue.stop_reason": "toolUse", - │ "flue.turn_purpose": "agent", - │ "model": "gpt-5.4-nano", - │ "provider": "openai" - │ } - │ metrics: { - │ "completion_tokens": 34, - │ "duration_ms": 0, - │ "estimated_cost": 0.00025350000000000004, - │ "prompt_cache_creation_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 1055, - │ "tokens": 1089 - │ } - ├── tool:web_search [tool] - │ input: { - │ "lookupId": "flue-session-2026", - │ "query": "Braintrust Flue reasoning stream instrumentation" - │ } - │ output: { - │ "content": [ - │ { - │ "text": "{\"lookupId\":\"flue-session-2026\",\"query\":\"Braintrust Flue reasoning stream instrumentation\",\"results\":[{\"title\":\"Flue reasoning stream instrumentation\",\"url\":\"https://example.test/flue/reasoning-streams\"}]}", - │ "type": "text" - │ } - │ ], - │ "details": { - │ "customTool": "web_search" - │ } - │ } - │ metadata: { - │ "flue.session": "main", - │ "flue.tool_name": "web_search", - │ "provider": "flue" - │ } - │ metrics: { - │ "duration_ms": 0 - │ } - ├── flue.turn [llm] - │ input: [ - │ { - │ "content": [ - │ { - │ "arguments": { - │ "lookupId": "flue-session-2026", - │ "query": "Braintrust Flue reasoning stream instrumentation" - │ }, - │ "id": "", - │ "name": "web_search", - │ "type": "toolCall" - │ } - │ ], - │ "role": "assistant" - │ }, - │ { - │ "content": [ - │ { - │ "text": "{\"lookupId\":\"flue-session-2026\",\"query\":\"Braintrust Flue reasoning stream instrumentation\",\"results\":[{\"title\":\"Flue reasoning stream instrumentation\",\"url\":\"https://example.test/flue/reasoning-streams\"}]}", - │ "type": "text" - │ } - │ ], - │ "isError": false, - │ "role": "toolResult", - │ "toolCallId": "", - │ "toolName": "web_search" - │ } - │ ] - │ output: { - │ "content": [ - │ { - │ "arguments": { - │ "url": "https://example.test/flue/reasoning-streams" - │ }, - │ "id": "", - │ "name": "summarize_source", - │ "type": "toolCall" - │ } - │ ], - │ "role": "assistant" - │ } - │ metadata: { - │ "flue.api": "openai-responses", - │ "flue.input_message_offset": 3, - │ "flue.input_mode": "delta", - │ "flue.model": "gpt-5.4-nano", - │ "flue.provider": "openai", - │ "flue.session": "main", - │ "flue.stop_reason": "toolUse", - │ "flue.turn_purpose": "agent", - │ "model": "gpt-5.4-nano", - │ "provider": "openai" - │ } - │ metrics: { - │ "completion_tokens": 30, - │ "duration_ms": 0, - │ "estimated_cost": 0.00026690000000000004, - │ "prompt_cache_creation_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 1147, - │ "tokens": 1177 - │ } - ├── tool:summarize_source [tool] - │ input: { - │ "url": "https://example.test/flue/reasoning-streams" - │ } - │ output: { - │ "content": [ - │ { - │ "text": "{\"summary\":\"Flue emits reasoning, tool execution, and LLM turn events separately.\",\"url\":\"https://example.test/flue/reasoning-streams\"}", - │ "type": "text" - │ } - │ ], - │ "details": { - │ "customTool": "summarize_source" - │ } - │ } - │ metadata: { - │ "flue.session": "main", - │ "flue.tool_name": "summarize_source", - │ "provider": "flue" - │ } - │ metrics: { - │ "duration_ms": 0 - │ } - └── flue.turn [llm] - input: [ - { - "content": [ - { - "arguments": { - "url": "https://example.test/flue/reasoning-streams" - }, - "id": "", - "name": "summarize_source", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "{\"summary\":\"Flue emits reasoning, tool execution, and LLM turn events separately.\",\"url\":\"https://example.test/flue/reasoning-streams\"}", - "type": "text" - } - ], - "isError": false, - "role": "toolResult", - "toolCallId": "", - "toolName": "summarize_source" - } - ] - output: { - "content": [ - { - "text": "PROMPT_DONE", - "textSignature": "{\"v\":1,\"id\":\"msg_045458052dc10379016a705b62dbd8819189b8287061debf12\",\"phase\":\"final_answer\"}", - "type": "text" - } - ], - "role": "assistant" - } - metadata: { - "flue.api": "openai-responses", - "flue.input_message_offset": 5, - "flue.input_mode": "delta", - "flue.model": "gpt-5.4-nano", - "flue.provider": "openai", - "flue.session": "main", - "flue.stop_reason": "stop", - "flue.turn_purpose": "agent", - "model": "gpt-5.4-nano", - "provider": "openai" - } - metrics: { - "completion_tokens": 7, - "duration_ms": 0, - "estimated_cost": 0.00025315000000000005, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 1222, - "tokens": 1229 - } diff --git a/e2e/scenarios/flue-instrumentation/__snapshots__/flue-v0-8-latest-explicit.span-tree.json b/e2e/scenarios/flue-instrumentation/__snapshots__/flue-v0-8-latest-explicit.span-tree.json deleted file mode 100644 index e26112b02..000000000 --- a/e2e/scenarios/flue-instrumentation/__snapshots__/flue-v0-8-latest-explicit.span-tree.json +++ /dev/null @@ -1,1662 +0,0 @@ -{ - "span_tree": [ - { - "name": "workflow:instrumentation", - "type": "task", - "children": [ - { - "name": "flue.skill", - "type": "task", - "children": [ - { - "name": "flue.turn", - "type": "llm", - "children": [ - { - "name": "flue.task", - "type": "task", - "children": [ - { - "name": "flue.turn", - "type": "llm", - "children": [], - "input": [ - { - "content": [ - { - "text": "Run the skill named \"e2e-flue-skill\" with arguments {\"marker\":\"SKILL_DONE\"}. Output only the marker and no extra text when finished.", - "type": "text" - } - ], - "role": "user" - } - ], - "output": { - "content": [ - { - "arguments": { - "agent": "e2e-flue-skill", - "cwd": ".", - "description": "Run e2e-flue-skill with marker SKILL_DONE", - "prompt": "Run the skill named \"e2e-flue-skill\" with arguments {\"marker\":\"SKILL_DONE\"}. Ensure completion and report the final output marker only." - }, - "id": "", - "name": "task", - "type": "toolCall" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "openai-responses", - "flue.input_mode": "full", - "flue.model": "gpt-5.4-nano", - "flue.provider": "openai", - "flue.session": "task:skill:", - "flue.stop_reason": "toolUse", - "flue.turn_purpose": "agent", - "model": "gpt-5.4-nano", - "provider": "openai" - }, - "metrics": { - "completion_tokens": 113, - "duration_ms": 0, - "estimated_cost": 0.00031225000000000003, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 855, - "tokens": 968 - } - }, - { - "name": "tool:task", - "type": "tool", - "children": [], - "input": { - "agent": "e2e-flue-skill", - "cwd": ".", - "description": "Run e2e-flue-skill with marker SKILL_DONE", - "prompt": "Run the skill named \"e2e-flue-skill\" with arguments {\"marker\":\"SKILL_DONE\"}. Ensure completion and report the final output marker only." - }, - "output": { - "content": [ - { - "text": "[flue] Subagent \"e2e-flue-skill\" is not declared. Available: (none).", - "type": "text" - } - ], - "details": {} - }, - "metadata": { - "flue.session": "task:skill:", - "flue.tool_name": "task", - "provider": "flue" - }, - "metrics": { - "duration_ms": 0 - }, - "error": "{\"content\":[{\"type\":\"text\",\"text\":\"[flue] Subagent \\\"e2e-flue-skill\\\" is not declared. Available: (none).\"}],\"details\":{}}" - }, - { - "name": "flue.turn", - "type": "llm", - "children": [], - "input": [ - { - "content": [ - { - "arguments": { - "agent": "e2e-flue-skill", - "cwd": ".", - "description": "Run e2e-flue-skill with marker SKILL_DONE", - "prompt": "Run the skill named \"e2e-flue-skill\" with arguments {\"marker\":\"SKILL_DONE\"}. Ensure completion and report the final output marker only." - }, - "id": "", - "name": "task", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "[flue] Subagent \"e2e-flue-skill\" is not declared. Available: (none).", - "type": "text" - } - ], - "isError": true, - "role": "toolResult", - "toolCallId": "", - "toolName": "task" - } - ], - "output": { - "content": [ - { - "arguments": { - "command": "find . -maxdepth 4 -name AGENTS.md -print", - "timeout": 100000 - }, - "id": "", - "name": "bash", - "type": "toolCall" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "openai-responses", - "flue.input_message_offset": 1, - "flue.input_mode": "delta", - "flue.model": "gpt-5.4-nano", - "flue.provider": "openai", - "flue.session": "task:skill:", - "flue.stop_reason": "toolUse", - "flue.turn_purpose": "agent", - "model": "gpt-5.4-nano", - "provider": "openai" - }, - "metrics": { - "completion_tokens": 94, - "duration_ms": 0, - "estimated_cost": 0.00031790000000000003, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 1002, - "tokens": 1096 - } - }, - { - "name": "tool:bash", - "type": "tool", - "children": [], - "input": { - "command": "find . -maxdepth 4 -name AGENTS.md -print", - "timeout": 100000 - }, - "output": { - "content": [ - { - "text": "(no output)", - "type": "text" - } - ], - "details": { - "command": "find . -maxdepth 4 -name AGENTS.md -print", - "exitCode": 0 - } - }, - "metadata": { - "flue.session": "task:skill:", - "flue.tool_name": "bash", - "provider": "flue" - }, - "metrics": { - "duration_ms": 0 - } - }, - { - "name": "flue.turn", - "type": "llm", - "children": [], - "input": [ - { - "content": [ - { - "arguments": { - "command": "find . -maxdepth 4 -name AGENTS.md -print", - "timeout": 100000 - }, - "id": "", - "name": "bash", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "(no output)", - "type": "text" - } - ], - "isError": false, - "role": "toolResult", - "toolCallId": "", - "toolName": "bash" - } - ], - "output": { - "content": [ - { - "arguments": { - "command": "find . -maxdepth 4 -type f -name AGENTS.md -print -exec sed -n '1,200p' {} \\;", - "timeout": 100000 - }, - "id": "", - "name": "bash", - "type": "toolCall" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "openai-responses", - "flue.input_message_offset": 3, - "flue.input_mode": "delta", - "flue.model": "gpt-5.4-nano", - "flue.provider": "openai", - "flue.session": "task:skill:", - "flue.stop_reason": "toolUse", - "flue.turn_purpose": "agent", - "model": "gpt-5.4-nano", - "provider": "openai" - }, - "metrics": { - "completion_tokens": 87, - "duration_ms": 0, - "estimated_cost": 0.00034815000000000003, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 1197, - "tokens": 1284 - } - }, - { - "name": "tool:bash", - "type": "tool", - "children": [], - "input": { - "command": "find . -maxdepth 4 -type f -name AGENTS.md -print -exec sed -n '1,200p' {} \\;", - "timeout": 100000 - }, - "output": { - "content": [ - { - "text": "(no output)", - "type": "text" - } - ], - "details": { - "command": "find . -maxdepth 4 -type f -name AGENTS.md -print -exec sed -n '1,200p' {} \\;", - "exitCode": 0 - } - }, - "metadata": { - "flue.session": "task:skill:", - "flue.tool_name": "bash", - "provider": "flue" - }, - "metrics": { - "duration_ms": 0 - } - }, - { - "name": "flue.turn", - "type": "llm", - "children": [], - "input": [ - { - "content": [ - { - "arguments": { - "command": "find . -maxdepth 4 -type f -name AGENTS.md -print -exec sed -n '1,200p' {} \\;", - "timeout": 100000 - }, - "id": "", - "name": "bash", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "(no output)", - "type": "text" - } - ], - "isError": false, - "role": "toolResult", - "toolCallId": "", - "toolName": "bash" - } - ], - "output": { - "content": [ - { - "arguments": { - "command": "find . -maxdepth 6 -type f -name 'AGENTS.md' -print", - "timeout": 100000 - }, - "id": "", - "name": "bash", - "type": "toolCall" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "openai-responses", - "flue.input_message_offset": 5, - "flue.input_mode": "delta", - "flue.model": "gpt-5.4-nano", - "flue.provider": "openai", - "flue.session": "task:skill:", - "flue.stop_reason": "toolUse", - "flue.turn_purpose": "agent", - "model": "gpt-5.4-nano", - "provider": "openai" - }, - "metrics": { - "completion_tokens": 58, - "duration_ms": 0, - "estimated_cost": 0.0003321, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 1298, - "tokens": 1356 - } - }, - { - "name": "tool:bash", - "type": "tool", - "children": [], - "input": { - "command": "find . -maxdepth 6 -type f -name 'AGENTS.md' -print", - "timeout": 100000 - }, - "output": { - "content": [ - { - "text": "(no output)", - "type": "text" - } - ], - "details": { - "command": "find . -maxdepth 6 -type f -name 'AGENTS.md' -print", - "exitCode": 0 - } - }, - "metadata": { - "flue.session": "task:skill:", - "flue.tool_name": "bash", - "provider": "flue" - }, - "metrics": { - "duration_ms": 0 - } - }, - { - "name": "flue.turn", - "type": "llm", - "children": [], - "input": [ - { - "content": [ - { - "arguments": { - "command": "find . -maxdepth 6 -type f -name 'AGENTS.md' -print", - "timeout": 100000 - }, - "id": "", - "name": "bash", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "(no output)", - "type": "text" - } - ], - "isError": false, - "role": "toolResult", - "toolCallId": "", - "toolName": "bash" - } - ], - "output": { - "content": [ - { - "arguments": { - "command": "find . -maxdepth 5 -type f -name '*flue*skill*' -o -name '*skill*' | head", - "timeout": 100000 - }, - "id": "", - "name": "bash", - "type": "toolCall" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "openai-responses", - "flue.input_message_offset": 7, - "flue.input_mode": "delta", - "flue.model": "gpt-5.4-nano", - "flue.provider": "openai", - "flue.session": "task:skill:", - "flue.stop_reason": "toolUse", - "flue.turn_purpose": "agent", - "model": "gpt-5.4-nano", - "provider": "openai" - }, - "metrics": { - "completion_tokens": 68, - "duration_ms": 0, - "estimated_cost": 0.00035900000000000005, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 1370, - "tokens": 1438 - } - }, - { - "name": "tool:bash", - "type": "tool", - "children": [], - "input": { - "command": "find . -maxdepth 5 -type f -name '*flue*skill*' -o -name '*skill*' | head", - "timeout": 100000 - }, - "output": { - "content": [ - { - "text": "./.agents/skills\n./.agents/skills/e2e-flue-skill", - "type": "text" - } - ], - "details": { - "command": "find . -maxdepth 5 -type f -name '*flue*skill*' -o -name '*skill*' | head", - "exitCode": 0 - } - }, - "metadata": { - "flue.session": "task:skill:", - "flue.tool_name": "bash", - "provider": "flue" - }, - "metrics": { - "duration_ms": 0 - } - }, - { - "name": "flue.turn", - "type": "llm", - "children": [], - "input": [ - { - "content": [ - { - "arguments": { - "command": "find . -maxdepth 5 -type f -name '*flue*skill*' -o -name '*skill*' | head", - "timeout": 100000 - }, - "id": "", - "name": "bash", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "./.agents/skills\n./.agents/skills/e2e-flue-skill", - "type": "text" - } - ], - "isError": false, - "role": "toolResult", - "toolCallId": "", - "toolName": "bash" - } - ], - "output": { - "content": [ - { - "arguments": { - "command": "find ./.agents/skills/e2e-flue-skill -maxdepth 2 -type f -print", - "timeout": 100000 - }, - "id": "", - "name": "bash", - "type": "toolCall" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "openai-responses", - "flue.input_message_offset": 9, - "flue.input_mode": "delta", - "flue.model": "gpt-5.4-nano", - "flue.provider": "openai", - "flue.session": "task:skill:", - "flue.stop_reason": "toolUse", - "flue.turn_purpose": "agent", - "model": "gpt-5.4-nano", - "provider": "openai" - }, - "metrics": { - "completion_tokens": 109, - "duration_ms": 0, - "estimated_cost": 0.00043885000000000007, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 1513, - "tokens": 1622 - } - }, - { - "name": "tool:bash", - "type": "tool", - "children": [], - "input": { - "command": "find ./.agents/skills/e2e-flue-skill -maxdepth 2 -type f -print", - "timeout": 100000 - }, - "output": { - "content": [ - { - "text": "./.agents/skills/e2e-flue-skill/SKILL.md", - "type": "text" - } - ], - "details": { - "command": "find ./.agents/skills/e2e-flue-skill -maxdepth 2 -type f -print", - "exitCode": 0 - } - }, - "metadata": { - "flue.session": "task:skill:", - "flue.tool_name": "bash", - "provider": "flue" - }, - "metrics": { - "duration_ms": 0 - } - }, - { - "name": "flue.turn", - "type": "llm", - "children": [], - "input": [ - { - "content": [ - { - "arguments": { - "command": "find ./.agents/skills/e2e-flue-skill -maxdepth 2 -type f -print", - "timeout": 100000 - }, - "id": "", - "name": "bash", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "./.agents/skills/e2e-flue-skill/SKILL.md", - "type": "text" - } - ], - "isError": false, - "role": "toolResult", - "toolCallId": "", - "toolName": "bash" - } - ], - "output": { - "content": [ - { - "arguments": { - "limit": 200, - "offset": 1, - "path": "./.agents/skills/e2e-flue-skill/SKILL.md" - }, - "id": "", - "name": "read", - "type": "toolCall" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "openai-responses", - "flue.input_message_offset": 11, - "flue.input_mode": "delta", - "flue.model": "gpt-5.4-nano", - "flue.provider": "openai", - "flue.session": "task:skill:", - "flue.stop_reason": "toolUse", - "flue.turn_purpose": "agent", - "model": "gpt-5.4-nano", - "provider": "openai" - }, - "metrics": { - "completion_tokens": 64, - "duration_ms": 0, - "estimated_cost": 0.00015592, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 1536, - "prompt_tokens": 226, - "tokens": 1826 - } - }, - { - "name": "tool:read", - "type": "tool", - "children": [], - "input": { - "limit": 200, - "offset": 1, - "path": "./.agents/skills/e2e-flue-skill/SKILL.md" - }, - "output": { - "content": [ - { - "text": "---\nname: e2e-flue-skill\ndescription: Deterministic skill used by the Braintrust Flue instrumentation e2e test.\n---\n\nReturn the marker from the provided args exactly once. Output no other text.\n", - "type": "text" - } - ], - "details": { - "lines": 7, - "path": "./.agents/skills/e2e-flue-skill/SKILL.md" - } - }, - "metadata": { - "flue.session": "task:skill:", - "flue.tool_name": "read", - "provider": "flue" - }, - "metrics": { - "duration_ms": 0 - } - }, - { - "name": "flue.turn", - "type": "llm", - "children": [], - "input": [ - { - "content": [ - { - "arguments": { - "limit": 200, - "offset": 1, - "path": "./.agents/skills/e2e-flue-skill/SKILL.md" - }, - "id": "", - "name": "read", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "---\nname: e2e-flue-skill\ndescription: Deterministic skill used by the Braintrust Flue instrumentation e2e test.\n---\n\nReturn the marker from the provided args exactly once. Output no other text.\n", - "type": "text" - } - ], - "isError": false, - "role": "toolResult", - "toolCallId": "", - "toolName": "read" - } - ], - "output": { - "content": [ - { - "arguments": { - "command": "cat ./package.json", - "timeout": 100000 - }, - "id": "", - "name": "bash", - "type": "toolCall" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "openai-responses", - "flue.input_message_offset": 13, - "flue.input_mode": "delta", - "flue.model": "gpt-5.4-nano", - "flue.provider": "openai", - "flue.session": "task:skill:", - "flue.stop_reason": "toolUse", - "flue.turn_purpose": "agent", - "model": "gpt-5.4-nano", - "provider": "openai" - }, - "metrics": { - "completion_tokens": 90, - "duration_ms": 0, - "estimated_cost": 0.00021242000000000003, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 1536, - "prompt_tokens": 346, - "tokens": 1972 - } - }, - { - "name": "tool:bash", - "type": "tool", - "children": [], - "input": { - "command": "cat ./package.json", - "timeout": 100000 - }, - "output": { - "content": [ - { - "text": "{\n \"name\": \"flue-instrumentation-scenario\",\n \"private\": true,\n \"type\": \"module\",\n \"braintrustScenario\": {\n \"bump\": {\n \"dependencies\": {\n \"flue-cli-v0-8-latest\": {\n \"package\": \"@flue/cli\",\n \"range\": \">=0.8.0 <0.9.0\"\n },\n \"flue-cli-v1-latest\": {\n \"package\": \"@flue/cli\",\n \"range\": \">=1.0.0-beta.3 <2.0.0-0\",\n \"allowPrerelease\": true\n },\n \"flue-runtime-v0-8-latest\": {\n \"package\": \"@flue/runtime\",\n \"range\": \">=0.8.0 <0.9.0\"\n },\n \"flue-runtime-v1-latest\": {\n \"package\": \"@flue/runtime\",\n \"range\": \">=1.0.0-beta.3 <2.0.0-0\",\n \"allowPrerelease\": true\n },\n \"flue-runtime-v2-latest\": {\n \"package\": \"@flue/runtime\",\n \"range\": \">=2.0.0 <3.0.0\"\n }\n }\n }\n },\n \"dependencies\": {\n \"@flue/cli\": \"0.8.0\",\n \"@flue/runtime\": \"0.8.0\",\n \"flue-cli-v0-8-latest\": \"npm:@flue/cli@0.8.1\",\n \"flue-cli-v1\": \"npm:@flue/cli@1.0.0-beta.3\",\n \"flue-cli-v1-latest\": \"npm:@flue/cli@1.0.0-beta.9\",\n \"flue-runtime-v0-8-latest\": \"npm:@flue/runtime@0.8.1\",\n \"flue-runtime-v1\": \"npm:@flue/runtime@1.0.0-beta.3\",\n \"flue-runtime-v1-latest\": \"npm:@flue/runtime@1.0.0-beta.9\",\n \"flue-runtime-v2\": \"npm:@flue/runtime@2.0.0\",\n \"flue-runtime-v2-latest\": \"npm:@flue/runtime@2.0.1\",\n \"pi-ai-v2\": \"npm:@earendil-works/pi-ai@0.83.0\",\n \"valibot\": \"1.4.2\"\n },\n \"pnpm\": {\n \"overrides\": {\n \"@flue/runtime@0.8.0>@earendil-works/pi-agent-core\": \"0.75.4\",\n \"@flue/runtime@0.8.0>@earendil-works/pi-ai\": \"0.75.4\",\n \"@flue/runtime@0.8.1>@earendil-works/pi-agent-core\": \"0.75.4\",\n \"@flue/runtime@0.8.1>@earendil-works/pi-ai\": \"0.75.4\",\n \"@hono/node-server@>=2.0.0 <2.0.10\": \"^2.0.10\",\n \"fast-uri@<3.1.5\": \"^3.1.5\",\n \"hono@>=4.0.0 <4.12.27\": \"^4.12.27\",\n \"brace-expansion@>=3.0.0 <5.0.9\": \"^5.0.9\",\n \"postcss@<8.5.18\": \"^8.5.18\",\n \"valibot@<1.4.2\": \"^1.4.2\",\n \"ip-address@<10.2.2\": \"^10.2.2\",\n \"undici@<7.29.0\": \"^7.29.0\"\n }\n }\n}", - "type": "text" - } - ], - "details": { - "command": "cat ./package.json", - "exitCode": 0 - } - }, - "metadata": { - "flue.session": "task:skill:", - "flue.tool_name": "bash", - "provider": "flue" - }, - "metrics": { - "duration_ms": 0 - } - }, - { - "name": "flue.turn", - "type": "llm", - "children": [], - "input": [ - { - "content": [ - { - "arguments": { - "command": "cat ./package.json", - "timeout": 100000 - }, - "id": "", - "name": "bash", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "{\n \"name\": \"flue-instrumentation-scenario\",\n \"private\": true,\n \"type\": \"module\",\n \"braintrustScenario\": {\n \"bump\": {\n \"dependencies\": {\n \"flue-cli-v0-8-latest\": {\n \"package\": \"@flue/cli\",\n \"range\": \">=0.8.0 <0.9.0\"\n },\n \"flue-cli-v1-latest\": {\n \"package\": \"@flue/cli\",\n \"range\": \">=1.0.0-beta.3 <2.0.0-0\",\n \"allowPrerelease\": true\n },\n \"flue-runtime-v0-8-latest\": {\n \"package\": \"@flue/runtime\",\n \"range\": \">=0.8.0 <0.9.0\"\n },\n \"flue-runtime-v1-latest\": {\n \"package\": \"@flue/runtime\",\n \"range\": \">=1.0.0-beta.3 <2.0.0-0\",\n \"allowPrerelease\": true\n },\n \"flue-runtime-v2-latest\": {\n \"package\": \"@flue/runtime\",\n \"range\": \">=2.0.0 <3.0.0\"\n }\n }\n }\n },\n \"dependencies\": {\n \"@flue/cli\": \"0.8.0\",\n \"@flue/runtime\": \"0.8.0\",\n \"flue-cli-v0-8-latest\": \"npm:@flue/cli@0.8.1\",\n \"flue-cli-v1\": \"npm:@flue/cli@1.0.0-beta.3\",\n \"flue-cli-v1-latest\": \"npm:@flue/cli@1.0.0-beta.9\",\n \"flue-runtime-v0-8-latest\": \"npm:@flue/runtime@0.8.1\",\n \"flue-runtime-v1\": \"npm:@flue/runtime@1.0.0-beta.3\",\n \"flue-runtime-v1-latest\": \"npm:@flue/runtime@1.0.0-beta.9\",\n \"flue-runtime-v2\": \"npm:@flue/runtime@2.0.0\",\n \"flue-runtime-v2-latest\": \"npm:@flue/runtime@2.0.1\",\n \"pi-ai-v2\": \"npm:@earendil-works/pi-ai@0.83.0\",\n \"valibot\": \"1.4.2\"\n },\n \"pnpm\": {\n \"overrides\": {\n \"@flue/runtime@0.8.0>@earendil-works/pi-agent-core\": \"0.75.4\",\n \"@flue/runtime@0.8.0>@earendil-works/pi-ai\": \"0.75.4\",\n \"@flue/runtime@0.8.1>@earendil-works/pi-agent-core\": \"0.75.4\",\n \"@flue/runtime@0.8.1>@earendil-works/pi-ai\": \"0.75.4\",\n \"@hono/node-server@>=2.0.0 <2.0.10\": \"^2.0.10\",\n \"fast-uri@<3.1.5\": \"^3.1.5\",\n \"hono@>=4.0.0 <4.12.27\": \"^4.12.27\",\n \"brace-expansion@>=3.0.0 <5.0.9\": \"^5.0.9\",\n \"postcss@<8.5.18\": \"^8.5.18\",\n \"valibot@<1.4.2\": \"^1.4.2\",\n \"ip-address@<10.2.2\": \"^10.2.2\",\n \"undici@<7.29.0\": \"^7.29.0\"\n }\n }\n}", - "type": "text" - } - ], - "isError": false, - "role": "toolResult", - "toolCallId": "", - "toolName": "bash" - } - ], - "output": { - "content": [ - { - "arguments": { - "command": "printf SKILL_DONE", - "timeout": 100000 - }, - "id": "", - "name": "bash", - "type": "toolCall" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "openai-responses", - "flue.input_message_offset": 15, - "flue.input_mode": "delta", - "flue.model": "gpt-5.4-nano", - "flue.provider": "openai", - "flue.session": "task:skill:", - "flue.stop_reason": "toolUse", - "flue.turn_purpose": "agent", - "model": "gpt-5.4-nano", - "provider": "openai" - }, - "metrics": { - "completion_tokens": 59, - "duration_ms": 0, - "estimated_cost": 0.0004814700000000001, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 1536, - "prompt_tokens": 1885, - "tokens": 3480 - } - }, - { - "name": "tool:bash", - "type": "tool", - "children": [], - "input": { - "command": "printf SKILL_DONE", - "timeout": 100000 - }, - "output": { - "content": [ - { - "text": "SKILL_DONE", - "type": "text" - } - ], - "details": { - "command": "printf SKILL_DONE", - "exitCode": 0 - } - }, - "metadata": { - "flue.session": "task:skill:", - "flue.tool_name": "bash", - "provider": "flue" - }, - "metrics": { - "duration_ms": 0 - } - }, - { - "name": "flue.turn", - "type": "llm", - "children": [], - "input": [ - { - "content": [ - { - "arguments": { - "command": "printf SKILL_DONE", - "timeout": 100000 - }, - "id": "", - "name": "bash", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "SKILL_DONE", - "type": "text" - } - ], - "isError": false, - "role": "toolResult", - "toolCallId": "", - "toolName": "bash" - } - ], - "output": { - "content": [ - { - "text": "SKILL_DONE", - "textSignature": "{\"v\":1,\"id\":\"msg_054a16a2f2ea3217016a705b70a41481a38ece40ecddf69512\",\"phase\":\"final_answer\"}", - "type": "text" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "openai-responses", - "flue.input_message_offset": 17, - "flue.input_mode": "delta", - "flue.model": "gpt-5.4-nano", - "flue.provider": "openai", - "flue.session": "task:skill:", - "flue.stop_reason": "stop", - "flue.turn_purpose": "agent", - "model": "gpt-5.4-nano", - "provider": "openai" - }, - "metrics": { - "completion_tokens": 7, - "duration_ms": 0, - "estimated_cost": 0.00031035000000000004, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 2560, - "prompt_tokens": 1252, - "tokens": 3819 - } - } - ], - "input": "Run the skill named \"e2e-flue-skill\" with arguments {\"marker\":\"SKILL_DONE\"}. Output only the marker and no extra text when finished.", - "output": "SKILL_DONE", - "metadata": { - "flue.session": "skill", - "provider": "flue" - }, - "metrics": { - "duration_ms": 0 - } - } - ], - "input": [ - { - "content": [ - { - "text": "Run the skill named \"e2e-flue-skill\".\n\nArguments:\n{\n \"marker\": \"SKILL_DONE\"\n}", - "type": "text" - } - ], - "role": "user" - } - ], - "output": { - "content": [ - { - "arguments": { - "cwd": ".", - "description": "Run the deterministic Flue instrumentation e2e test skill e2e-flue-skill.", - "prompt": "Run the skill named \"e2e-flue-skill\" with arguments {\"marker\":\"SKILL_DONE\"}. Output only the marker and no extra text when finished." - }, - "id": "", - "name": "task", - "type": "toolCall" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "openai-responses", - "flue.model": "gpt-5.4-nano", - "flue.provider": "openai", - "flue.session": "skill", - "flue.stop_reason": "toolUse", - "flue.turn_purpose": "agent", - "model": "gpt-5.4-nano", - "provider": "openai" - }, - "metrics": { - "completion_tokens": 109, - "duration_ms": 0, - "estimated_cost": 0.00030565000000000003, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 847, - "tokens": 956 - } - }, - { - "name": "tool:task", - "type": "tool", - "children": [], - "input": { - "cwd": ".", - "description": "Run the deterministic Flue instrumentation e2e test skill e2e-flue-skill.", - "prompt": "Run the skill named \"e2e-flue-skill\" with arguments {\"marker\":\"SKILL_DONE\"}. Output only the marker and no extra text when finished." - }, - "output": { - "content": [ - { - "text": "SKILL_DONE", - "type": "text" - } - ], - "details": { - "cwd": ".", - "messageId": "", - "session": "task:skill:", - "taskId": "" - } - }, - "metadata": { - "flue.session": "skill", - "flue.tool_name": "task", - "provider": "flue" - }, - "metrics": { - "duration_ms": 0 - } - }, - { - "name": "flue.turn", - "type": "llm", - "children": [], - "input": [ - { - "content": [ - { - "text": "Run the skill named \"e2e-flue-skill\".\n\nArguments:\n{\n \"marker\": \"SKILL_DONE\"\n}", - "type": "text" - } - ], - "role": "user" - }, - { - "content": [ - { - "arguments": { - "cwd": ".", - "description": "Run the deterministic Flue instrumentation e2e test skill e2e-flue-skill.", - "prompt": "Run the skill named \"e2e-flue-skill\" with arguments {\"marker\":\"SKILL_DONE\"}. Output only the marker and no extra text when finished." - }, - "id": "", - "name": "task", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "SKILL_DONE", - "type": "text" - } - ], - "isError": false, - "role": "toolResult", - "toolCallId": "", - "toolName": "task" - } - ], - "output": { - "content": [ - { - "text": "SKILL_DONE", - "textSignature": "{\"v\":1,\"id\":\"msg_0f099b3fce0721fd016a705b71558881a38c53a767e09d9e31\",\"phase\":\"final_answer\"}", - "type": "text" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "openai-responses", - "flue.model": "gpt-5.4-nano", - "flue.provider": "openai", - "flue.session": "skill", - "flue.stop_reason": "stop", - "flue.turn_purpose": "agent", - "model": "gpt-5.4-nano", - "provider": "openai" - }, - "metrics": { - "completion_tokens": 7, - "duration_ms": 0, - "estimated_cost": 0.00020255000000000002, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 969, - "tokens": 976 - } - } - ], - "input": [ - { - "content": [ - { - "text": "Run the skill named \"e2e-flue-skill\".\n\nArguments:\n{\n \"marker\": \"SKILL_DONE\"\n}", - "type": "text" - } - ], - "role": "user" - } - ], - "output": "SKILL_DONE", - "metadata": { - "flue.operation": "skill", - "flue.session": "skill", - "flue.workflow_name": "instrumentation", - "provider": "flue", - "scenario": "flue-instrumentation" - }, - "metrics": { - "duration_ms": 0 - } - }, - { - "name": "flue.task", - "type": "task", - "children": [ - { - "name": "flue.turn", - "type": "llm", - "children": [], - "input": [ - { - "content": [ - { - "text": "Reply with exactly TASK_DONE and no other text.", - "type": "text" - } - ], - "role": "user" - } - ], - "output": { - "content": [ - { - "text": "TASK_DONE", - "textSignature": "{\"v\":1,\"id\":\"msg_0f3adfdb1510c165016a705b72843c81a0b30d242da683a882\"}", - "type": "text" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "openai-responses", - "flue.input_mode": "full", - "flue.model": "gpt-4o-mini", - "flue.provider": "openai", - "flue.session": "task:task:", - "flue.stop_reason": "stop", - "flue.turn_purpose": "agent", - "model": "gpt-4o-mini", - "provider": "openai" - }, - "metrics": { - "completion_tokens": 4, - "duration_ms": 0, - "estimated_cost": 0.0001263, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 826, - "tokens": 830 - } - } - ], - "input": "Reply with exactly TASK_DONE and no other text.", - "output": "TASK_DONE", - "metadata": { - "flue.session": "task", - "provider": "flue" - }, - "metrics": { - "duration_ms": 0 - } - }, - { - "name": "flue.compact", - "type": "task", - "children": [ - { - "name": "compaction:manual", - "type": "task", - "children": [ - { - "name": "flue.turn", - "type": "llm", - "children": [], - "input": [ - { - "content": [ - { - "text": "\n[User]: Complete this instrumented research flow. Call exactly one tool per turn and wait for each tool result before choosing the next tool. Step 1: call lookup with query \"flue instrumentation\". Step 2: use the lookup result id as lookupId and call web_search with query \"Braintrust Flue reasoning stream instrumentation\". Step 3: use the first web_search result url and call summarize_source. After summarize_source returns, reply with exactly PROMPT_DONE and no other text.\n\n[Assistant tool calls]: lookup(query=\"flue instrumentation\")\n\n[Tool result]: {\"id\":\"flue-session-2026\",\"query\":\"flue instrumentation\",\"topic\":\"session instrumentation\"}\n\n[Assistant tool calls]: web_search(lookupId=\"flue-session-2026\", query=\"Braintrust Flue reasoning stream instrumentation\")\n\n[Tool result]: {\"lookupId\":\"flue-session-2026\",\"query\":\"Braintrust Flue reasoning stream instrumentation\",\"results\":[{\"title\":\"Flue reasoning stream instrumentation\",\"url\":\"https://example.test/flue/reasoning-streams\"}]}\n\n[Assistant tool calls]: summarize_source(url=\"https://example.test/flue/reasoning-streams\")\n\n[Tool result]: {\"summary\":\"Flue emits reasoning, tool execution, and LLM turn events separately.\",\"url\":\"https://example.test/flue/reasoning-streams\"}\n\n\nThis is the PREFIX of a turn that was too large to keep. The SUFFIX (recent work) is retained.\n\nSummarize the prefix to provide context for the retained suffix:\n\n## Original Request\n[What did the user ask for in this turn?]\n\n## Early Progress\n- [Key decisions and work done in the prefix]\n\n## Context for Suffix\n- [Information needed to understand the retained recent work]\n\nBe concise. Focus on what's needed to understand the kept suffix.", - "type": "text" - } - ], - "role": "user" - } - ], - "output": { - "content": [ - { - "text": "## Original Request\nThe user requested a step-by-step execution of an instrumented research flow involving tool calls to gather information on \"flue instrumentation\" and \"", - "textSignature": "{\"v\":1,\"id\":\"msg_04d28ba03840bb65016a705b7358a081a38e69464057840036\"}", - "type": "text" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "openai-responses", - "flue.model": "gpt-4o-mini", - "flue.provider": "openai", - "flue.session": "main", - "flue.stop_reason": "stop", - "flue.turn_purpose": "compaction_prefix", - "model": "gpt-4o-mini", - "provider": "openai" - }, - "metrics": { - "completion_tokens": 0, - "duration_ms": 0, - "estimated_cost": 0, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 0, - "tokens": 0 - } - } - ], - "input": { - "estimatedTokens": 1229, - "reason": "manual" - }, - "output": { - "messagesAfter": 2, - "messagesBefore": 8 - }, - "metadata": { - "flue.compaction_reason": "manual", - "flue.session": "main", - "provider": "flue" - }, - "metrics": { - "duration_ms": 0, - "messages_after": 2, - "messages_before": 8 - } - } - ], - "input": { - "estimatedTokens": 1229, - "reason": "manual" - }, - "output": { - "completed": true - }, - "metadata": { - "flue.operation": "compact", - "flue.session": "main", - "flue.workflow_name": "instrumentation", - "provider": "flue", - "scenario": "flue-instrumentation" - }, - "metrics": { - "duration_ms": 0 - } - } - ], - "input": { - "metadata": { - "scenario": "flue-instrumentation", - "testRunId": "" - }, - "scenario": "flue-instrumentation" - }, - "output": { - "scenario": "flue-instrumentation", - "status": "done" - }, - "metadata": { - "flue.workflow_name": "instrumentation", - "provider": "flue", - "scenario": "flue-instrumentation" - }, - "metrics": { - "duration_ms": 0 - } - }, - { - "name": "flue.prompt", - "type": "task", - "children": [ - { - "name": "flue.turn", - "type": "llm", - "children": [], - "input": [ - { - "content": [ - { - "text": "Complete this instrumented research flow. Call exactly one tool per turn and wait for each tool result before choosing the next tool. Step 1: call lookup with query \"flue instrumentation\". Step 2: use the lookup result id as lookupId and call web_search with query \"Braintrust Flue reasoning stream instrumentation\". Step 3: use the first web_search result url and call summarize_source. After summarize_source returns, reply with exactly PROMPT_DONE and no other text.", - "type": "text" - } - ], - "role": "user" - } - ], - "output": { - "content": [ - { - "arguments": { - "query": "flue instrumentation" - }, - "id": "", - "name": "lookup", - "type": "toolCall" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "openai-responses", - "flue.input_mode": "full", - "flue.model": "gpt-5.4-nano", - "flue.provider": "openai", - "flue.session": "main", - "flue.stop_reason": "toolUse", - "flue.turn_purpose": "agent", - "model": "gpt-5.4-nano", - "provider": "openai" - }, - "metrics": { - "completion_tokens": 19, - "duration_ms": 0, - "estimated_cost": 0.00022475000000000001, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 1005, - "tokens": 1024 - } - }, - { - "name": "tool:lookup", - "type": "tool", - "children": [], - "input": { - "query": "flue instrumentation" - }, - "output": { - "content": [ - { - "text": "{\"id\":\"flue-session-2026\",\"query\":\"flue instrumentation\",\"topic\":\"session instrumentation\"}", - "type": "text" - } - ], - "details": { - "customTool": "lookup" - } - }, - "metadata": { - "flue.session": "main", - "flue.tool_name": "lookup", - "provider": "flue" - }, - "metrics": { - "duration_ms": 0 - } - }, - { - "name": "flue.turn", - "type": "llm", - "children": [], - "input": [ - { - "content": [ - { - "arguments": { - "query": "flue instrumentation" - }, - "id": "", - "name": "lookup", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "{\"id\":\"flue-session-2026\",\"query\":\"flue instrumentation\",\"topic\":\"session instrumentation\"}", - "type": "text" - } - ], - "isError": false, - "role": "toolResult", - "toolCallId": "", - "toolName": "lookup" - } - ], - "output": { - "content": [ - { - "arguments": { - "lookupId": "flue-session-2026", - "query": "Braintrust Flue reasoning stream instrumentation" - }, - "id": "", - "name": "web_search", - "type": "toolCall" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "openai-responses", - "flue.input_message_offset": 1, - "flue.input_mode": "delta", - "flue.model": "gpt-5.4-nano", - "flue.provider": "openai", - "flue.session": "main", - "flue.stop_reason": "toolUse", - "flue.turn_purpose": "agent", - "model": "gpt-5.4-nano", - "provider": "openai" - }, - "metrics": { - "completion_tokens": 34, - "duration_ms": 0, - "estimated_cost": 0.00025350000000000004, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 1055, - "tokens": 1089 - } - }, - { - "name": "tool:web_search", - "type": "tool", - "children": [], - "input": { - "lookupId": "flue-session-2026", - "query": "Braintrust Flue reasoning stream instrumentation" - }, - "output": { - "content": [ - { - "text": "{\"lookupId\":\"flue-session-2026\",\"query\":\"Braintrust Flue reasoning stream instrumentation\",\"results\":[{\"title\":\"Flue reasoning stream instrumentation\",\"url\":\"https://example.test/flue/reasoning-streams\"}]}", - "type": "text" - } - ], - "details": { - "customTool": "web_search" - } - }, - "metadata": { - "flue.session": "main", - "flue.tool_name": "web_search", - "provider": "flue" - }, - "metrics": { - "duration_ms": 0 - } - }, - { - "name": "flue.turn", - "type": "llm", - "children": [], - "input": [ - { - "content": [ - { - "arguments": { - "lookupId": "flue-session-2026", - "query": "Braintrust Flue reasoning stream instrumentation" - }, - "id": "", - "name": "web_search", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "{\"lookupId\":\"flue-session-2026\",\"query\":\"Braintrust Flue reasoning stream instrumentation\",\"results\":[{\"title\":\"Flue reasoning stream instrumentation\",\"url\":\"https://example.test/flue/reasoning-streams\"}]}", - "type": "text" - } - ], - "isError": false, - "role": "toolResult", - "toolCallId": "", - "toolName": "web_search" - } - ], - "output": { - "content": [ - { - "arguments": { - "url": "https://example.test/flue/reasoning-streams" - }, - "id": "", - "name": "summarize_source", - "type": "toolCall" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "openai-responses", - "flue.input_message_offset": 3, - "flue.input_mode": "delta", - "flue.model": "gpt-5.4-nano", - "flue.provider": "openai", - "flue.session": "main", - "flue.stop_reason": "toolUse", - "flue.turn_purpose": "agent", - "model": "gpt-5.4-nano", - "provider": "openai" - }, - "metrics": { - "completion_tokens": 30, - "duration_ms": 0, - "estimated_cost": 0.00026690000000000004, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 1147, - "tokens": 1177 - } - }, - { - "name": "tool:summarize_source", - "type": "tool", - "children": [], - "input": { - "url": "https://example.test/flue/reasoning-streams" - }, - "output": { - "content": [ - { - "text": "{\"summary\":\"Flue emits reasoning, tool execution, and LLM turn events separately.\",\"url\":\"https://example.test/flue/reasoning-streams\"}", - "type": "text" - } - ], - "details": { - "customTool": "summarize_source" - } - }, - "metadata": { - "flue.session": "main", - "flue.tool_name": "summarize_source", - "provider": "flue" - }, - "metrics": { - "duration_ms": 0 - } - }, - { - "name": "flue.turn", - "type": "llm", - "children": [], - "input": [ - { - "content": [ - { - "arguments": { - "url": "https://example.test/flue/reasoning-streams" - }, - "id": "", - "name": "summarize_source", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "{\"summary\":\"Flue emits reasoning, tool execution, and LLM turn events separately.\",\"url\":\"https://example.test/flue/reasoning-streams\"}", - "type": "text" - } - ], - "isError": false, - "role": "toolResult", - "toolCallId": "", - "toolName": "summarize_source" - } - ], - "output": { - "content": [ - { - "text": "PROMPT_DONE", - "textSignature": "{\"v\":1,\"id\":\"msg_045458052dc10379016a705b62dbd8819189b8287061debf12\",\"phase\":\"final_answer\"}", - "type": "text" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "openai-responses", - "flue.input_message_offset": 5, - "flue.input_mode": "delta", - "flue.model": "gpt-5.4-nano", - "flue.provider": "openai", - "flue.session": "main", - "flue.stop_reason": "stop", - "flue.turn_purpose": "agent", - "model": "gpt-5.4-nano", - "provider": "openai" - }, - "metrics": { - "completion_tokens": 7, - "duration_ms": 0, - "estimated_cost": 0.00025315000000000005, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 1222, - "tokens": 1229 - } - } - ], - "input": [ - { - "content": [ - { - "text": "Complete this instrumented research flow. Call exactly one tool per turn and wait for each tool result before choosing the next tool. Step 1: call lookup with query \"flue instrumentation\". Step 2: use the lookup result id as lookupId and call web_search with query \"Braintrust Flue reasoning stream instrumentation\". Step 3: use the first web_search result url and call summarize_source. After summarize_source returns, reply with exactly PROMPT_DONE and no other text.", - "type": "text" - } - ], - "role": "user" - } - ], - "output": "PROMPT_DONE", - "metadata": { - "flue.operation": "prompt", - "flue.session": "main", - "flue.workflow_name": "instrumentation", - "provider": "flue", - "scenario": "flue-instrumentation" - }, - "metrics": { - "duration_ms": 0 - } - } - ] -} diff --git a/e2e/scenarios/flue-instrumentation/__snapshots__/flue-v0-8-latest-explicit.span-tree.txt b/e2e/scenarios/flue-instrumentation/__snapshots__/flue-v0-8-latest-explicit.span-tree.txt deleted file mode 100644 index abf057e06..000000000 --- a/e2e/scenarios/flue-instrumentation/__snapshots__/flue-v0-8-latest-explicit.span-tree.txt +++ /dev/null @@ -1,1499 +0,0 @@ -span_tree: -├── workflow:instrumentation [task] -│ input: { -│ "metadata": { -│ "scenario": "flue-instrumentation", -│ "testRunId": "" -│ }, -│ "scenario": "flue-instrumentation" -│ } -│ output: { -│ "scenario": "flue-instrumentation", -│ "status": "done" -│ } -│ metadata: { -│ "flue.workflow_name": "instrumentation", -│ "provider": "flue", -│ "scenario": "flue-instrumentation" -│ } -│ metrics: { -│ "duration_ms": 0 -│ } -│ ├── flue.skill [task] -│ │ input: [ -│ │ { -│ │ "content": [ -│ │ { -│ │ "text": "Run the skill named \"e2e-flue-skill\".\n\nArguments:\n{\n \"marker\": \"SKILL_DONE\"\n}", -│ │ "type": "text" -│ │ } -│ │ ], -│ │ "role": "user" -│ │ } -│ │ ] -│ │ output: "SKILL_DONE" -│ │ metadata: { -│ │ "flue.operation": "skill", -│ │ "flue.session": "skill", -│ │ "flue.workflow_name": "instrumentation", -│ │ "provider": "flue", -│ │ "scenario": "flue-instrumentation" -│ │ } -│ │ metrics: { -│ │ "duration_ms": 0 -│ │ } -│ │ ├── flue.turn [llm] -│ │ │ input: [ -│ │ │ { -│ │ │ "content": [ -│ │ │ { -│ │ │ "text": "Run the skill named \"e2e-flue-skill\".\n\nArguments:\n{\n \"marker\": \"SKILL_DONE\"\n}", -│ │ │ "type": "text" -│ │ │ } -│ │ │ ], -│ │ │ "role": "user" -│ │ │ } -│ │ │ ] -│ │ │ output: { -│ │ │ "content": [ -│ │ │ { -│ │ │ "arguments": { -│ │ │ "cwd": ".", -│ │ │ "description": "Run the deterministic Flue instrumentation e2e test skill e2e-flue-skill.", -│ │ │ "prompt": "Run the skill named \"e2e-flue-skill\" with arguments {\"marker\":\"SKILL_DONE\"}. Output only the marker and no extra text when finished." -│ │ │ }, -│ │ │ "id": "", -│ │ │ "name": "task", -│ │ │ "type": "toolCall" -│ │ │ } -│ │ │ ], -│ │ │ "role": "assistant" -│ │ │ } -│ │ │ metadata: { -│ │ │ "flue.api": "openai-responses", -│ │ │ "flue.model": "gpt-5.4-nano", -│ │ │ "flue.provider": "openai", -│ │ │ "flue.session": "skill", -│ │ │ "flue.stop_reason": "toolUse", -│ │ │ "flue.turn_purpose": "agent", -│ │ │ "model": "gpt-5.4-nano", -│ │ │ "provider": "openai" -│ │ │ } -│ │ │ metrics: { -│ │ │ "completion_tokens": 109, -│ │ │ "duration_ms": 0, -│ │ │ "estimated_cost": 0.00030565000000000003, -│ │ │ "prompt_cache_creation_tokens": 0, -│ │ │ "prompt_cached_tokens": 0, -│ │ │ "prompt_tokens": 847, -│ │ │ "tokens": 956 -│ │ │ } -│ │ │ └── flue.task [task] -│ │ │ input: "Run the skill named \"e2e-flue-skill\" with arguments {\"marker\":\"SKILL_DONE\"}. Output only the marker and no extra text when finished." -│ │ │ output: "SKILL_DONE" -│ │ │ metadata: { -│ │ │ "flue.session": "skill", -│ │ │ "provider": "flue" -│ │ │ } -│ │ │ metrics: { -│ │ │ "duration_ms": 0 -│ │ │ } -│ │ │ ├── flue.turn [llm] -│ │ │ │ input: [ -│ │ │ │ { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "text": "Run the skill named \"e2e-flue-skill\" with arguments {\"marker\":\"SKILL_DONE\"}. Output only the marker and no extra text when finished.", -│ │ │ │ "type": "text" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "role": "user" -│ │ │ │ } -│ │ │ │ ] -│ │ │ │ output: { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "arguments": { -│ │ │ │ "agent": "e2e-flue-skill", -│ │ │ │ "cwd": ".", -│ │ │ │ "description": "Run e2e-flue-skill with marker SKILL_DONE", -│ │ │ │ "prompt": "Run the skill named \"e2e-flue-skill\" with arguments {\"marker\":\"SKILL_DONE\"}. Ensure completion and report the final output marker only." -│ │ │ │ }, -│ │ │ │ "id": "", -│ │ │ │ "name": "task", -│ │ │ │ "type": "toolCall" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "role": "assistant" -│ │ │ │ } -│ │ │ │ metadata: { -│ │ │ │ "flue.api": "openai-responses", -│ │ │ │ "flue.input_mode": "full", -│ │ │ │ "flue.model": "gpt-5.4-nano", -│ │ │ │ "flue.provider": "openai", -│ │ │ │ "flue.session": "task:skill:", -│ │ │ │ "flue.stop_reason": "toolUse", -│ │ │ │ "flue.turn_purpose": "agent", -│ │ │ │ "model": "gpt-5.4-nano", -│ │ │ │ "provider": "openai" -│ │ │ │ } -│ │ │ │ metrics: { -│ │ │ │ "completion_tokens": 113, -│ │ │ │ "duration_ms": 0, -│ │ │ │ "estimated_cost": 0.00031225000000000003, -│ │ │ │ "prompt_cache_creation_tokens": 0, -│ │ │ │ "prompt_cached_tokens": 0, -│ │ │ │ "prompt_tokens": 855, -│ │ │ │ "tokens": 968 -│ │ │ │ } -│ │ │ ├── tool:task [tool] -│ │ │ │ input: { -│ │ │ │ "agent": "e2e-flue-skill", -│ │ │ │ "cwd": ".", -│ │ │ │ "description": "Run e2e-flue-skill with marker SKILL_DONE", -│ │ │ │ "prompt": "Run the skill named \"e2e-flue-skill\" with arguments {\"marker\":\"SKILL_DONE\"}. Ensure completion and report the final output marker only." -│ │ │ │ } -│ │ │ │ output: { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "text": "[flue] Subagent \"e2e-flue-skill\" is not declared. Available: (none).", -│ │ │ │ "type": "text" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "details": {} -│ │ │ │ } -│ │ │ │ metadata: { -│ │ │ │ "flue.session": "task:skill:", -│ │ │ │ "flue.tool_name": "task", -│ │ │ │ "provider": "flue" -│ │ │ │ } -│ │ │ │ metrics: { -│ │ │ │ "duration_ms": 0 -│ │ │ │ } -│ │ │ │ error: "{\"content\":[{\"type\":\"text\",\"text\":\"[flue] Subagent \\\"e2e-flue-skill\\\" is not declared. Available: (none).\"}],\"details\":{}}" -│ │ │ ├── flue.turn [llm] -│ │ │ │ input: [ -│ │ │ │ { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "arguments": { -│ │ │ │ "agent": "e2e-flue-skill", -│ │ │ │ "cwd": ".", -│ │ │ │ "description": "Run e2e-flue-skill with marker SKILL_DONE", -│ │ │ │ "prompt": "Run the skill named \"e2e-flue-skill\" with arguments {\"marker\":\"SKILL_DONE\"}. Ensure completion and report the final output marker only." -│ │ │ │ }, -│ │ │ │ "id": "", -│ │ │ │ "name": "task", -│ │ │ │ "type": "toolCall" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "role": "assistant" -│ │ │ │ }, -│ │ │ │ { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "text": "[flue] Subagent \"e2e-flue-skill\" is not declared. Available: (none).", -│ │ │ │ "type": "text" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "isError": true, -│ │ │ │ "role": "toolResult", -│ │ │ │ "toolCallId": "", -│ │ │ │ "toolName": "task" -│ │ │ │ } -│ │ │ │ ] -│ │ │ │ output: { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "arguments": { -│ │ │ │ "command": "find . -maxdepth 4 -name AGENTS.md -print", -│ │ │ │ "timeout": 100000 -│ │ │ │ }, -│ │ │ │ "id": "", -│ │ │ │ "name": "bash", -│ │ │ │ "type": "toolCall" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "role": "assistant" -│ │ │ │ } -│ │ │ │ metadata: { -│ │ │ │ "flue.api": "openai-responses", -│ │ │ │ "flue.input_message_offset": 1, -│ │ │ │ "flue.input_mode": "delta", -│ │ │ │ "flue.model": "gpt-5.4-nano", -│ │ │ │ "flue.provider": "openai", -│ │ │ │ "flue.session": "task:skill:", -│ │ │ │ "flue.stop_reason": "toolUse", -│ │ │ │ "flue.turn_purpose": "agent", -│ │ │ │ "model": "gpt-5.4-nano", -│ │ │ │ "provider": "openai" -│ │ │ │ } -│ │ │ │ metrics: { -│ │ │ │ "completion_tokens": 94, -│ │ │ │ "duration_ms": 0, -│ │ │ │ "estimated_cost": 0.00031790000000000003, -│ │ │ │ "prompt_cache_creation_tokens": 0, -│ │ │ │ "prompt_cached_tokens": 0, -│ │ │ │ "prompt_tokens": 1002, -│ │ │ │ "tokens": 1096 -│ │ │ │ } -│ │ │ ├── tool:bash [tool] -│ │ │ │ input: { -│ │ │ │ "command": "find . -maxdepth 4 -name AGENTS.md -print", -│ │ │ │ "timeout": 100000 -│ │ │ │ } -│ │ │ │ output: { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "text": "(no output)", -│ │ │ │ "type": "text" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "details": { -│ │ │ │ "command": "find . -maxdepth 4 -name AGENTS.md -print", -│ │ │ │ "exitCode": 0 -│ │ │ │ } -│ │ │ │ } -│ │ │ │ metadata: { -│ │ │ │ "flue.session": "task:skill:", -│ │ │ │ "flue.tool_name": "bash", -│ │ │ │ "provider": "flue" -│ │ │ │ } -│ │ │ │ metrics: { -│ │ │ │ "duration_ms": 0 -│ │ │ │ } -│ │ │ ├── flue.turn [llm] -│ │ │ │ input: [ -│ │ │ │ { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "arguments": { -│ │ │ │ "command": "find . -maxdepth 4 -name AGENTS.md -print", -│ │ │ │ "timeout": 100000 -│ │ │ │ }, -│ │ │ │ "id": "", -│ │ │ │ "name": "bash", -│ │ │ │ "type": "toolCall" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "role": "assistant" -│ │ │ │ }, -│ │ │ │ { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "text": "(no output)", -│ │ │ │ "type": "text" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "isError": false, -│ │ │ │ "role": "toolResult", -│ │ │ │ "toolCallId": "", -│ │ │ │ "toolName": "bash" -│ │ │ │ } -│ │ │ │ ] -│ │ │ │ output: { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "arguments": { -│ │ │ │ "command": "find . -maxdepth 4 -type f -name AGENTS.md -print -exec sed -n '1,200p' {} \\;", -│ │ │ │ "timeout": 100000 -│ │ │ │ }, -│ │ │ │ "id": "", -│ │ │ │ "name": "bash", -│ │ │ │ "type": "toolCall" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "role": "assistant" -│ │ │ │ } -│ │ │ │ metadata: { -│ │ │ │ "flue.api": "openai-responses", -│ │ │ │ "flue.input_message_offset": 3, -│ │ │ │ "flue.input_mode": "delta", -│ │ │ │ "flue.model": "gpt-5.4-nano", -│ │ │ │ "flue.provider": "openai", -│ │ │ │ "flue.session": "task:skill:", -│ │ │ │ "flue.stop_reason": "toolUse", -│ │ │ │ "flue.turn_purpose": "agent", -│ │ │ │ "model": "gpt-5.4-nano", -│ │ │ │ "provider": "openai" -│ │ │ │ } -│ │ │ │ metrics: { -│ │ │ │ "completion_tokens": 87, -│ │ │ │ "duration_ms": 0, -│ │ │ │ "estimated_cost": 0.00034815000000000003, -│ │ │ │ "prompt_cache_creation_tokens": 0, -│ │ │ │ "prompt_cached_tokens": 0, -│ │ │ │ "prompt_tokens": 1197, -│ │ │ │ "tokens": 1284 -│ │ │ │ } -│ │ │ ├── tool:bash [tool] -│ │ │ │ input: { -│ │ │ │ "command": "find . -maxdepth 4 -type f -name AGENTS.md -print -exec sed -n '1,200p' {} \\;", -│ │ │ │ "timeout": 100000 -│ │ │ │ } -│ │ │ │ output: { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "text": "(no output)", -│ │ │ │ "type": "text" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "details": { -│ │ │ │ "command": "find . -maxdepth 4 -type f -name AGENTS.md -print -exec sed -n '1,200p' {} \\;", -│ │ │ │ "exitCode": 0 -│ │ │ │ } -│ │ │ │ } -│ │ │ │ metadata: { -│ │ │ │ "flue.session": "task:skill:", -│ │ │ │ "flue.tool_name": "bash", -│ │ │ │ "provider": "flue" -│ │ │ │ } -│ │ │ │ metrics: { -│ │ │ │ "duration_ms": 0 -│ │ │ │ } -│ │ │ ├── flue.turn [llm] -│ │ │ │ input: [ -│ │ │ │ { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "arguments": { -│ │ │ │ "command": "find . -maxdepth 4 -type f -name AGENTS.md -print -exec sed -n '1,200p' {} \\;", -│ │ │ │ "timeout": 100000 -│ │ │ │ }, -│ │ │ │ "id": "", -│ │ │ │ "name": "bash", -│ │ │ │ "type": "toolCall" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "role": "assistant" -│ │ │ │ }, -│ │ │ │ { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "text": "(no output)", -│ │ │ │ "type": "text" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "isError": false, -│ │ │ │ "role": "toolResult", -│ │ │ │ "toolCallId": "", -│ │ │ │ "toolName": "bash" -│ │ │ │ } -│ │ │ │ ] -│ │ │ │ output: { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "arguments": { -│ │ │ │ "command": "find . -maxdepth 6 -type f -name 'AGENTS.md' -print", -│ │ │ │ "timeout": 100000 -│ │ │ │ }, -│ │ │ │ "id": "", -│ │ │ │ "name": "bash", -│ │ │ │ "type": "toolCall" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "role": "assistant" -│ │ │ │ } -│ │ │ │ metadata: { -│ │ │ │ "flue.api": "openai-responses", -│ │ │ │ "flue.input_message_offset": 5, -│ │ │ │ "flue.input_mode": "delta", -│ │ │ │ "flue.model": "gpt-5.4-nano", -│ │ │ │ "flue.provider": "openai", -│ │ │ │ "flue.session": "task:skill:", -│ │ │ │ "flue.stop_reason": "toolUse", -│ │ │ │ "flue.turn_purpose": "agent", -│ │ │ │ "model": "gpt-5.4-nano", -│ │ │ │ "provider": "openai" -│ │ │ │ } -│ │ │ │ metrics: { -│ │ │ │ "completion_tokens": 58, -│ │ │ │ "duration_ms": 0, -│ │ │ │ "estimated_cost": 0.0003321, -│ │ │ │ "prompt_cache_creation_tokens": 0, -│ │ │ │ "prompt_cached_tokens": 0, -│ │ │ │ "prompt_tokens": 1298, -│ │ │ │ "tokens": 1356 -│ │ │ │ } -│ │ │ ├── tool:bash [tool] -│ │ │ │ input: { -│ │ │ │ "command": "find . -maxdepth 6 -type f -name 'AGENTS.md' -print", -│ │ │ │ "timeout": 100000 -│ │ │ │ } -│ │ │ │ output: { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "text": "(no output)", -│ │ │ │ "type": "text" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "details": { -│ │ │ │ "command": "find . -maxdepth 6 -type f -name 'AGENTS.md' -print", -│ │ │ │ "exitCode": 0 -│ │ │ │ } -│ │ │ │ } -│ │ │ │ metadata: { -│ │ │ │ "flue.session": "task:skill:", -│ │ │ │ "flue.tool_name": "bash", -│ │ │ │ "provider": "flue" -│ │ │ │ } -│ │ │ │ metrics: { -│ │ │ │ "duration_ms": 0 -│ │ │ │ } -│ │ │ ├── flue.turn [llm] -│ │ │ │ input: [ -│ │ │ │ { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "arguments": { -│ │ │ │ "command": "find . -maxdepth 6 -type f -name 'AGENTS.md' -print", -│ │ │ │ "timeout": 100000 -│ │ │ │ }, -│ │ │ │ "id": "", -│ │ │ │ "name": "bash", -│ │ │ │ "type": "toolCall" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "role": "assistant" -│ │ │ │ }, -│ │ │ │ { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "text": "(no output)", -│ │ │ │ "type": "text" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "isError": false, -│ │ │ │ "role": "toolResult", -│ │ │ │ "toolCallId": "", -│ │ │ │ "toolName": "bash" -│ │ │ │ } -│ │ │ │ ] -│ │ │ │ output: { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "arguments": { -│ │ │ │ "command": "find . -maxdepth 5 -type f -name '*flue*skill*' -o -name '*skill*' | head", -│ │ │ │ "timeout": 100000 -│ │ │ │ }, -│ │ │ │ "id": "", -│ │ │ │ "name": "bash", -│ │ │ │ "type": "toolCall" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "role": "assistant" -│ │ │ │ } -│ │ │ │ metadata: { -│ │ │ │ "flue.api": "openai-responses", -│ │ │ │ "flue.input_message_offset": 7, -│ │ │ │ "flue.input_mode": "delta", -│ │ │ │ "flue.model": "gpt-5.4-nano", -│ │ │ │ "flue.provider": "openai", -│ │ │ │ "flue.session": "task:skill:", -│ │ │ │ "flue.stop_reason": "toolUse", -│ │ │ │ "flue.turn_purpose": "agent", -│ │ │ │ "model": "gpt-5.4-nano", -│ │ │ │ "provider": "openai" -│ │ │ │ } -│ │ │ │ metrics: { -│ │ │ │ "completion_tokens": 68, -│ │ │ │ "duration_ms": 0, -│ │ │ │ "estimated_cost": 0.00035900000000000005, -│ │ │ │ "prompt_cache_creation_tokens": 0, -│ │ │ │ "prompt_cached_tokens": 0, -│ │ │ │ "prompt_tokens": 1370, -│ │ │ │ "tokens": 1438 -│ │ │ │ } -│ │ │ ├── tool:bash [tool] -│ │ │ │ input: { -│ │ │ │ "command": "find . -maxdepth 5 -type f -name '*flue*skill*' -o -name '*skill*' | head", -│ │ │ │ "timeout": 100000 -│ │ │ │ } -│ │ │ │ output: { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "text": "./.agents/skills\n./.agents/skills/e2e-flue-skill", -│ │ │ │ "type": "text" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "details": { -│ │ │ │ "command": "find . -maxdepth 5 -type f -name '*flue*skill*' -o -name '*skill*' | head", -│ │ │ │ "exitCode": 0 -│ │ │ │ } -│ │ │ │ } -│ │ │ │ metadata: { -│ │ │ │ "flue.session": "task:skill:", -│ │ │ │ "flue.tool_name": "bash", -│ │ │ │ "provider": "flue" -│ │ │ │ } -│ │ │ │ metrics: { -│ │ │ │ "duration_ms": 0 -│ │ │ │ } -│ │ │ ├── flue.turn [llm] -│ │ │ │ input: [ -│ │ │ │ { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "arguments": { -│ │ │ │ "command": "find . -maxdepth 5 -type f -name '*flue*skill*' -o -name '*skill*' | head", -│ │ │ │ "timeout": 100000 -│ │ │ │ }, -│ │ │ │ "id": "", -│ │ │ │ "name": "bash", -│ │ │ │ "type": "toolCall" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "role": "assistant" -│ │ │ │ }, -│ │ │ │ { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "text": "./.agents/skills\n./.agents/skills/e2e-flue-skill", -│ │ │ │ "type": "text" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "isError": false, -│ │ │ │ "role": "toolResult", -│ │ │ │ "toolCallId": "", -│ │ │ │ "toolName": "bash" -│ │ │ │ } -│ │ │ │ ] -│ │ │ │ output: { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "arguments": { -│ │ │ │ "command": "find ./.agents/skills/e2e-flue-skill -maxdepth 2 -type f -print", -│ │ │ │ "timeout": 100000 -│ │ │ │ }, -│ │ │ │ "id": "", -│ │ │ │ "name": "bash", -│ │ │ │ "type": "toolCall" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "role": "assistant" -│ │ │ │ } -│ │ │ │ metadata: { -│ │ │ │ "flue.api": "openai-responses", -│ │ │ │ "flue.input_message_offset": 9, -│ │ │ │ "flue.input_mode": "delta", -│ │ │ │ "flue.model": "gpt-5.4-nano", -│ │ │ │ "flue.provider": "openai", -│ │ │ │ "flue.session": "task:skill:", -│ │ │ │ "flue.stop_reason": "toolUse", -│ │ │ │ "flue.turn_purpose": "agent", -│ │ │ │ "model": "gpt-5.4-nano", -│ │ │ │ "provider": "openai" -│ │ │ │ } -│ │ │ │ metrics: { -│ │ │ │ "completion_tokens": 109, -│ │ │ │ "duration_ms": 0, -│ │ │ │ "estimated_cost": 0.00043885000000000007, -│ │ │ │ "prompt_cache_creation_tokens": 0, -│ │ │ │ "prompt_cached_tokens": 0, -│ │ │ │ "prompt_tokens": 1513, -│ │ │ │ "tokens": 1622 -│ │ │ │ } -│ │ │ ├── tool:bash [tool] -│ │ │ │ input: { -│ │ │ │ "command": "find ./.agents/skills/e2e-flue-skill -maxdepth 2 -type f -print", -│ │ │ │ "timeout": 100000 -│ │ │ │ } -│ │ │ │ output: { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "text": "./.agents/skills/e2e-flue-skill/SKILL.md", -│ │ │ │ "type": "text" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "details": { -│ │ │ │ "command": "find ./.agents/skills/e2e-flue-skill -maxdepth 2 -type f -print", -│ │ │ │ "exitCode": 0 -│ │ │ │ } -│ │ │ │ } -│ │ │ │ metadata: { -│ │ │ │ "flue.session": "task:skill:", -│ │ │ │ "flue.tool_name": "bash", -│ │ │ │ "provider": "flue" -│ │ │ │ } -│ │ │ │ metrics: { -│ │ │ │ "duration_ms": 0 -│ │ │ │ } -│ │ │ ├── flue.turn [llm] -│ │ │ │ input: [ -│ │ │ │ { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "arguments": { -│ │ │ │ "command": "find ./.agents/skills/e2e-flue-skill -maxdepth 2 -type f -print", -│ │ │ │ "timeout": 100000 -│ │ │ │ }, -│ │ │ │ "id": "", -│ │ │ │ "name": "bash", -│ │ │ │ "type": "toolCall" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "role": "assistant" -│ │ │ │ }, -│ │ │ │ { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "text": "./.agents/skills/e2e-flue-skill/SKILL.md", -│ │ │ │ "type": "text" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "isError": false, -│ │ │ │ "role": "toolResult", -│ │ │ │ "toolCallId": "", -│ │ │ │ "toolName": "bash" -│ │ │ │ } -│ │ │ │ ] -│ │ │ │ output: { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "arguments": { -│ │ │ │ "limit": 200, -│ │ │ │ "offset": 1, -│ │ │ │ "path": "./.agents/skills/e2e-flue-skill/SKILL.md" -│ │ │ │ }, -│ │ │ │ "id": "", -│ │ │ │ "name": "read", -│ │ │ │ "type": "toolCall" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "role": "assistant" -│ │ │ │ } -│ │ │ │ metadata: { -│ │ │ │ "flue.api": "openai-responses", -│ │ │ │ "flue.input_message_offset": 11, -│ │ │ │ "flue.input_mode": "delta", -│ │ │ │ "flue.model": "gpt-5.4-nano", -│ │ │ │ "flue.provider": "openai", -│ │ │ │ "flue.session": "task:skill:", -│ │ │ │ "flue.stop_reason": "toolUse", -│ │ │ │ "flue.turn_purpose": "agent", -│ │ │ │ "model": "gpt-5.4-nano", -│ │ │ │ "provider": "openai" -│ │ │ │ } -│ │ │ │ metrics: { -│ │ │ │ "completion_tokens": 64, -│ │ │ │ "duration_ms": 0, -│ │ │ │ "estimated_cost": 0.00015592, -│ │ │ │ "prompt_cache_creation_tokens": 0, -│ │ │ │ "prompt_cached_tokens": 1536, -│ │ │ │ "prompt_tokens": 226, -│ │ │ │ "tokens": 1826 -│ │ │ │ } -│ │ │ ├── tool:read [tool] -│ │ │ │ input: { -│ │ │ │ "limit": 200, -│ │ │ │ "offset": 1, -│ │ │ │ "path": "./.agents/skills/e2e-flue-skill/SKILL.md" -│ │ │ │ } -│ │ │ │ output: { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "text": "---\nname: e2e-flue-skill\ndescription: Deterministic skill used by the Braintrust Flue instrumentation e2e test.\n---\n\nReturn the marker from the provided args exactly once. Output no other text.\n", -│ │ │ │ "type": "text" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "details": { -│ │ │ │ "lines": 7, -│ │ │ │ "path": "./.agents/skills/e2e-flue-skill/SKILL.md" -│ │ │ │ } -│ │ │ │ } -│ │ │ │ metadata: { -│ │ │ │ "flue.session": "task:skill:", -│ │ │ │ "flue.tool_name": "read", -│ │ │ │ "provider": "flue" -│ │ │ │ } -│ │ │ │ metrics: { -│ │ │ │ "duration_ms": 0 -│ │ │ │ } -│ │ │ ├── flue.turn [llm] -│ │ │ │ input: [ -│ │ │ │ { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "arguments": { -│ │ │ │ "limit": 200, -│ │ │ │ "offset": 1, -│ │ │ │ "path": "./.agents/skills/e2e-flue-skill/SKILL.md" -│ │ │ │ }, -│ │ │ │ "id": "", -│ │ │ │ "name": "read", -│ │ │ │ "type": "toolCall" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "role": "assistant" -│ │ │ │ }, -│ │ │ │ { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "text": "---\nname: e2e-flue-skill\ndescription: Deterministic skill used by the Braintrust Flue instrumentation e2e test.\n---\n\nReturn the marker from the provided args exactly once. Output no other text.\n", -│ │ │ │ "type": "text" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "isError": false, -│ │ │ │ "role": "toolResult", -│ │ │ │ "toolCallId": "", -│ │ │ │ "toolName": "read" -│ │ │ │ } -│ │ │ │ ] -│ │ │ │ output: { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "arguments": { -│ │ │ │ "command": "cat ./package.json", -│ │ │ │ "timeout": 100000 -│ │ │ │ }, -│ │ │ │ "id": "", -│ │ │ │ "name": "bash", -│ │ │ │ "type": "toolCall" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "role": "assistant" -│ │ │ │ } -│ │ │ │ metadata: { -│ │ │ │ "flue.api": "openai-responses", -│ │ │ │ "flue.input_message_offset": 13, -│ │ │ │ "flue.input_mode": "delta", -│ │ │ │ "flue.model": "gpt-5.4-nano", -│ │ │ │ "flue.provider": "openai", -│ │ │ │ "flue.session": "task:skill:", -│ │ │ │ "flue.stop_reason": "toolUse", -│ │ │ │ "flue.turn_purpose": "agent", -│ │ │ │ "model": "gpt-5.4-nano", -│ │ │ │ "provider": "openai" -│ │ │ │ } -│ │ │ │ metrics: { -│ │ │ │ "completion_tokens": 90, -│ │ │ │ "duration_ms": 0, -│ │ │ │ "estimated_cost": 0.00021242000000000003, -│ │ │ │ "prompt_cache_creation_tokens": 0, -│ │ │ │ "prompt_cached_tokens": 1536, -│ │ │ │ "prompt_tokens": 346, -│ │ │ │ "tokens": 1972 -│ │ │ │ } -│ │ │ ├── tool:bash [tool] -│ │ │ │ input: { -│ │ │ │ "command": "cat ./package.json", -│ │ │ │ "timeout": 100000 -│ │ │ │ } -│ │ │ │ output: { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "text": "{\n \"name\": \"flue-instrumentation-scenario\",\n \"private\": true,\n \"type\": \"module\",\n \"braintrustScenario\": {\n \"bump\": {\n \"dependencies\": {\n \"flue-cli-v0-8-latest\": {\n \"package\": \"@flue/cli\",\n \"range\": \">=0.8.0 <0.9.0\"\n },\n \"flue-cli-v1-latest\": {\n \"package\": \"@flue/cli\",\n \"range\": \">=1.0.0-beta.3 <2.0.0-0\",\n \"allowPrerelease\": true\n },\n \"flue-runtime-v0-8-latest\": {\n \"package\": \"@flue/runtime\",\n \"range\": \">=0.8.0 <0.9.0\"\n },\n \"flue-runtime-v1-latest\": {\n \"package\": \"@flue/runtime\",\n \"range\": \">=1.0.0-beta.3 <2.0.0-0\",\n \"allowPrerelease\": true\n },\n \"flue-runtime-v2-latest\": {\n \"package\": \"@flue/runtime\",\n \"range\": \">=2.0.0 <3.0.0\"\n }\n }\n }\n },\n \"dependencies\": {\n \"@flue/cli\": \"0.8.0\",\n \"@flue/runtime\": \"0.8.0\",\n \"flue-cli-v0-8-latest\": \"npm:@flue/cli@0.8.1\",\n \"flue-cli-v1\": \"npm:@flue/cli@1.0.0-beta.3\",\n \"flue-cli-v1-latest\": \"npm:@flue/cli@1.0.0-beta.9\",\n \"flue-runtime-v0-8-latest\": \"npm:@flue/runtime@0.8.1\",\n \"flue-runtime-v1\": \"npm:@flue/runtime@1.0.0-beta.3\",\n \"flue-runtime-v1-latest\": \"npm:@flue/runtime@1.0.0-beta.9\",\n \"flue-runtime-v2\": \"npm:@flue/runtime@2.0.0\",\n \"flue-runtime-v2-latest\": \"npm:@flue/runtime@2.0.1\",\n \"pi-ai-v2\": \"npm:@earendil-works/pi-ai@0.83.0\",\n \"valibot\": \"1.4.2\"\n },\n \"pnpm\": {\n \"overrides\": {\n \"@flue/runtime@0.8.0>@earendil-works/pi-agent-core\": \"0.75.4\",\n \"@flue/runtime@0.8.0>@earendil-works/pi-ai\": \"0.75.4\",\n \"@flue/runtime@0.8.1>@earendil-works/pi-agent-core\": \"0.75.4\",\n \"@flue/runtime@0.8.1>@earendil-works/pi-ai\": \"0.75.4\",\n \"@hono/node-server@>=2.0.0 <2.0.10\": \"^2.0.10\",\n \"fast-uri@<3.1.5\": \"^3.1.5\",\n \"hono@>=4.0.0 <4.12.27\": \"^4.12.27\",\n \"brace-expansion@>=3.0.0 <5.0.9\": \"^5.0.9\",\n \"postcss@<8.5.18\": \"^8.5.18\",\n \"valibot@<1.4.2\": \"^1.4.2\",\n \"ip-address@<10.2.2\": \"^10.2.2\",\n \"undici@<7.29.0\": \"^7.29.0\"\n }\n }\n}", -│ │ │ │ "type": "text" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "details": { -│ │ │ │ "command": "cat ./package.json", -│ │ │ │ "exitCode": 0 -│ │ │ │ } -│ │ │ │ } -│ │ │ │ metadata: { -│ │ │ │ "flue.session": "task:skill:", -│ │ │ │ "flue.tool_name": "bash", -│ │ │ │ "provider": "flue" -│ │ │ │ } -│ │ │ │ metrics: { -│ │ │ │ "duration_ms": 0 -│ │ │ │ } -│ │ │ ├── flue.turn [llm] -│ │ │ │ input: [ -│ │ │ │ { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "arguments": { -│ │ │ │ "command": "cat ./package.json", -│ │ │ │ "timeout": 100000 -│ │ │ │ }, -│ │ │ │ "id": "", -│ │ │ │ "name": "bash", -│ │ │ │ "type": "toolCall" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "role": "assistant" -│ │ │ │ }, -│ │ │ │ { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "text": "{\n \"name\": \"flue-instrumentation-scenario\",\n \"private\": true,\n \"type\": \"module\",\n \"braintrustScenario\": {\n \"bump\": {\n \"dependencies\": {\n \"flue-cli-v0-8-latest\": {\n \"package\": \"@flue/cli\",\n \"range\": \">=0.8.0 <0.9.0\"\n },\n \"flue-cli-v1-latest\": {\n \"package\": \"@flue/cli\",\n \"range\": \">=1.0.0-beta.3 <2.0.0-0\",\n \"allowPrerelease\": true\n },\n \"flue-runtime-v0-8-latest\": {\n \"package\": \"@flue/runtime\",\n \"range\": \">=0.8.0 <0.9.0\"\n },\n \"flue-runtime-v1-latest\": {\n \"package\": \"@flue/runtime\",\n \"range\": \">=1.0.0-beta.3 <2.0.0-0\",\n \"allowPrerelease\": true\n },\n \"flue-runtime-v2-latest\": {\n \"package\": \"@flue/runtime\",\n \"range\": \">=2.0.0 <3.0.0\"\n }\n }\n }\n },\n \"dependencies\": {\n \"@flue/cli\": \"0.8.0\",\n \"@flue/runtime\": \"0.8.0\",\n \"flue-cli-v0-8-latest\": \"npm:@flue/cli@0.8.1\",\n \"flue-cli-v1\": \"npm:@flue/cli@1.0.0-beta.3\",\n \"flue-cli-v1-latest\": \"npm:@flue/cli@1.0.0-beta.9\",\n \"flue-runtime-v0-8-latest\": \"npm:@flue/runtime@0.8.1\",\n \"flue-runtime-v1\": \"npm:@flue/runtime@1.0.0-beta.3\",\n \"flue-runtime-v1-latest\": \"npm:@flue/runtime@1.0.0-beta.9\",\n \"flue-runtime-v2\": \"npm:@flue/runtime@2.0.0\",\n \"flue-runtime-v2-latest\": \"npm:@flue/runtime@2.0.1\",\n \"pi-ai-v2\": \"npm:@earendil-works/pi-ai@0.83.0\",\n \"valibot\": \"1.4.2\"\n },\n \"pnpm\": {\n \"overrides\": {\n \"@flue/runtime@0.8.0>@earendil-works/pi-agent-core\": \"0.75.4\",\n \"@flue/runtime@0.8.0>@earendil-works/pi-ai\": \"0.75.4\",\n \"@flue/runtime@0.8.1>@earendil-works/pi-agent-core\": \"0.75.4\",\n \"@flue/runtime@0.8.1>@earendil-works/pi-ai\": \"0.75.4\",\n \"@hono/node-server@>=2.0.0 <2.0.10\": \"^2.0.10\",\n \"fast-uri@<3.1.5\": \"^3.1.5\",\n \"hono@>=4.0.0 <4.12.27\": \"^4.12.27\",\n \"brace-expansion@>=3.0.0 <5.0.9\": \"^5.0.9\",\n \"postcss@<8.5.18\": \"^8.5.18\",\n \"valibot@<1.4.2\": \"^1.4.2\",\n \"ip-address@<10.2.2\": \"^10.2.2\",\n \"undici@<7.29.0\": \"^7.29.0\"\n }\n }\n}", -│ │ │ │ "type": "text" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "isError": false, -│ │ │ │ "role": "toolResult", -│ │ │ │ "toolCallId": "", -│ │ │ │ "toolName": "bash" -│ │ │ │ } -│ │ │ │ ] -│ │ │ │ output: { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "arguments": { -│ │ │ │ "command": "printf SKILL_DONE", -│ │ │ │ "timeout": 100000 -│ │ │ │ }, -│ │ │ │ "id": "", -│ │ │ │ "name": "bash", -│ │ │ │ "type": "toolCall" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "role": "assistant" -│ │ │ │ } -│ │ │ │ metadata: { -│ │ │ │ "flue.api": "openai-responses", -│ │ │ │ "flue.input_message_offset": 15, -│ │ │ │ "flue.input_mode": "delta", -│ │ │ │ "flue.model": "gpt-5.4-nano", -│ │ │ │ "flue.provider": "openai", -│ │ │ │ "flue.session": "task:skill:", -│ │ │ │ "flue.stop_reason": "toolUse", -│ │ │ │ "flue.turn_purpose": "agent", -│ │ │ │ "model": "gpt-5.4-nano", -│ │ │ │ "provider": "openai" -│ │ │ │ } -│ │ │ │ metrics: { -│ │ │ │ "completion_tokens": 59, -│ │ │ │ "duration_ms": 0, -│ │ │ │ "estimated_cost": 0.0004814700000000001, -│ │ │ │ "prompt_cache_creation_tokens": 0, -│ │ │ │ "prompt_cached_tokens": 1536, -│ │ │ │ "prompt_tokens": 1885, -│ │ │ │ "tokens": 3480 -│ │ │ │ } -│ │ │ ├── tool:bash [tool] -│ │ │ │ input: { -│ │ │ │ "command": "printf SKILL_DONE", -│ │ │ │ "timeout": 100000 -│ │ │ │ } -│ │ │ │ output: { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "text": "SKILL_DONE", -│ │ │ │ "type": "text" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "details": { -│ │ │ │ "command": "printf SKILL_DONE", -│ │ │ │ "exitCode": 0 -│ │ │ │ } -│ │ │ │ } -│ │ │ │ metadata: { -│ │ │ │ "flue.session": "task:skill:", -│ │ │ │ "flue.tool_name": "bash", -│ │ │ │ "provider": "flue" -│ │ │ │ } -│ │ │ │ metrics: { -│ │ │ │ "duration_ms": 0 -│ │ │ │ } -│ │ │ └── flue.turn [llm] -│ │ │ input: [ -│ │ │ { -│ │ │ "content": [ -│ │ │ { -│ │ │ "arguments": { -│ │ │ "command": "printf SKILL_DONE", -│ │ │ "timeout": 100000 -│ │ │ }, -│ │ │ "id": "", -│ │ │ "name": "bash", -│ │ │ "type": "toolCall" -│ │ │ } -│ │ │ ], -│ │ │ "role": "assistant" -│ │ │ }, -│ │ │ { -│ │ │ "content": [ -│ │ │ { -│ │ │ "text": "SKILL_DONE", -│ │ │ "type": "text" -│ │ │ } -│ │ │ ], -│ │ │ "isError": false, -│ │ │ "role": "toolResult", -│ │ │ "toolCallId": "", -│ │ │ "toolName": "bash" -│ │ │ } -│ │ │ ] -│ │ │ output: { -│ │ │ "content": [ -│ │ │ { -│ │ │ "text": "SKILL_DONE", -│ │ │ "textSignature": "{\"v\":1,\"id\":\"msg_054a16a2f2ea3217016a705b70a41481a38ece40ecddf69512\",\"phase\":\"final_answer\"}", -│ │ │ "type": "text" -│ │ │ } -│ │ │ ], -│ │ │ "role": "assistant" -│ │ │ } -│ │ │ metadata: { -│ │ │ "flue.api": "openai-responses", -│ │ │ "flue.input_message_offset": 17, -│ │ │ "flue.input_mode": "delta", -│ │ │ "flue.model": "gpt-5.4-nano", -│ │ │ "flue.provider": "openai", -│ │ │ "flue.session": "task:skill:", -│ │ │ "flue.stop_reason": "stop", -│ │ │ "flue.turn_purpose": "agent", -│ │ │ "model": "gpt-5.4-nano", -│ │ │ "provider": "openai" -│ │ │ } -│ │ │ metrics: { -│ │ │ "completion_tokens": 7, -│ │ │ "duration_ms": 0, -│ │ │ "estimated_cost": 0.00031035000000000004, -│ │ │ "prompt_cache_creation_tokens": 0, -│ │ │ "prompt_cached_tokens": 2560, -│ │ │ "prompt_tokens": 1252, -│ │ │ "tokens": 3819 -│ │ │ } -│ │ ├── tool:task [tool] -│ │ │ input: { -│ │ │ "cwd": ".", -│ │ │ "description": "Run the deterministic Flue instrumentation e2e test skill e2e-flue-skill.", -│ │ │ "prompt": "Run the skill named \"e2e-flue-skill\" with arguments {\"marker\":\"SKILL_DONE\"}. Output only the marker and no extra text when finished." -│ │ │ } -│ │ │ output: { -│ │ │ "content": [ -│ │ │ { -│ │ │ "text": "SKILL_DONE", -│ │ │ "type": "text" -│ │ │ } -│ │ │ ], -│ │ │ "details": { -│ │ │ "cwd": ".", -│ │ │ "messageId": "", -│ │ │ "session": "task:skill:", -│ │ │ "taskId": "" -│ │ │ } -│ │ │ } -│ │ │ metadata: { -│ │ │ "flue.session": "skill", -│ │ │ "flue.tool_name": "task", -│ │ │ "provider": "flue" -│ │ │ } -│ │ │ metrics: { -│ │ │ "duration_ms": 0 -│ │ │ } -│ │ └── flue.turn [llm] -│ │ input: [ -│ │ { -│ │ "content": [ -│ │ { -│ │ "text": "Run the skill named \"e2e-flue-skill\".\n\nArguments:\n{\n \"marker\": \"SKILL_DONE\"\n}", -│ │ "type": "text" -│ │ } -│ │ ], -│ │ "role": "user" -│ │ }, -│ │ { -│ │ "content": [ -│ │ { -│ │ "arguments": { -│ │ "cwd": ".", -│ │ "description": "Run the deterministic Flue instrumentation e2e test skill e2e-flue-skill.", -│ │ "prompt": "Run the skill named \"e2e-flue-skill\" with arguments {\"marker\":\"SKILL_DONE\"}. Output only the marker and no extra text when finished." -│ │ }, -│ │ "id": "", -│ │ "name": "task", -│ │ "type": "toolCall" -│ │ } -│ │ ], -│ │ "role": "assistant" -│ │ }, -│ │ { -│ │ "content": [ -│ │ { -│ │ "text": "SKILL_DONE", -│ │ "type": "text" -│ │ } -│ │ ], -│ │ "isError": false, -│ │ "role": "toolResult", -│ │ "toolCallId": "", -│ │ "toolName": "task" -│ │ } -│ │ ] -│ │ output: { -│ │ "content": [ -│ │ { -│ │ "text": "SKILL_DONE", -│ │ "textSignature": "{\"v\":1,\"id\":\"msg_0f099b3fce0721fd016a705b71558881a38c53a767e09d9e31\",\"phase\":\"final_answer\"}", -│ │ "type": "text" -│ │ } -│ │ ], -│ │ "role": "assistant" -│ │ } -│ │ metadata: { -│ │ "flue.api": "openai-responses", -│ │ "flue.model": "gpt-5.4-nano", -│ │ "flue.provider": "openai", -│ │ "flue.session": "skill", -│ │ "flue.stop_reason": "stop", -│ │ "flue.turn_purpose": "agent", -│ │ "model": "gpt-5.4-nano", -│ │ "provider": "openai" -│ │ } -│ │ metrics: { -│ │ "completion_tokens": 7, -│ │ "duration_ms": 0, -│ │ "estimated_cost": 0.00020255000000000002, -│ │ "prompt_cache_creation_tokens": 0, -│ │ "prompt_cached_tokens": 0, -│ │ "prompt_tokens": 969, -│ │ "tokens": 976 -│ │ } -│ ├── flue.task [task] -│ │ input: "Reply with exactly TASK_DONE and no other text." -│ │ output: "TASK_DONE" -│ │ metadata: { -│ │ "flue.session": "task", -│ │ "provider": "flue" -│ │ } -│ │ metrics: { -│ │ "duration_ms": 0 -│ │ } -│ │ └── flue.turn [llm] -│ │ input: [ -│ │ { -│ │ "content": [ -│ │ { -│ │ "text": "Reply with exactly TASK_DONE and no other text.", -│ │ "type": "text" -│ │ } -│ │ ], -│ │ "role": "user" -│ │ } -│ │ ] -│ │ output: { -│ │ "content": [ -│ │ { -│ │ "text": "TASK_DONE", -│ │ "textSignature": "{\"v\":1,\"id\":\"msg_0f3adfdb1510c165016a705b72843c81a0b30d242da683a882\"}", -│ │ "type": "text" -│ │ } -│ │ ], -│ │ "role": "assistant" -│ │ } -│ │ metadata: { -│ │ "flue.api": "openai-responses", -│ │ "flue.input_mode": "full", -│ │ "flue.model": "gpt-4o-mini", -│ │ "flue.provider": "openai", -│ │ "flue.session": "task:task:", -│ │ "flue.stop_reason": "stop", -│ │ "flue.turn_purpose": "agent", -│ │ "model": "gpt-4o-mini", -│ │ "provider": "openai" -│ │ } -│ │ metrics: { -│ │ "completion_tokens": 4, -│ │ "duration_ms": 0, -│ │ "estimated_cost": 0.0001263, -│ │ "prompt_cache_creation_tokens": 0, -│ │ "prompt_cached_tokens": 0, -│ │ "prompt_tokens": 826, -│ │ "tokens": 830 -│ │ } -│ └── flue.compact [task] -│ input: { -│ "estimatedTokens": 1229, -│ "reason": "manual" -│ } -│ output: { -│ "completed": true -│ } -│ metadata: { -│ "flue.operation": "compact", -│ "flue.session": "main", -│ "flue.workflow_name": "instrumentation", -│ "provider": "flue", -│ "scenario": "flue-instrumentation" -│ } -│ metrics: { -│ "duration_ms": 0 -│ } -│ └── compaction:manual [task] -│ input: { -│ "estimatedTokens": 1229, -│ "reason": "manual" -│ } -│ output: { -│ "messagesAfter": 2, -│ "messagesBefore": 8 -│ } -│ metadata: { -│ "flue.compaction_reason": "manual", -│ "flue.session": "main", -│ "provider": "flue" -│ } -│ metrics: { -│ "duration_ms": 0, -│ "messages_after": 2, -│ "messages_before": 8 -│ } -│ └── flue.turn [llm] -│ input: [ -│ { -│ "content": [ -│ { -│ "text": "\n[User]: Complete this instrumented research flow. Call exactly one tool per turn and wait for each tool result before choosing the next tool. Step 1: call lookup with query \"flue instrumentation\". Step 2: use the lookup result id as lookupId and call web_search with query \"Braintrust Flue reasoning stream instrumentation\". Step 3: use the first web_search result url and call summarize_source. After summarize_source returns, reply with exactly PROMPT_DONE and no other text.\n\n[Assistant tool calls]: lookup(query=\"flue instrumentation\")\n\n[Tool result]: {\"id\":\"flue-session-2026\",\"query\":\"flue instrumentation\",\"topic\":\"session instrumentation\"}\n\n[Assistant tool calls]: web_search(lookupId=\"flue-session-2026\", query=\"Braintrust Flue reasoning stream instrumentation\")\n\n[Tool result]: {\"lookupId\":\"flue-session-2026\",\"query\":\"Braintrust Flue reasoning stream instrumentation\",\"results\":[{\"title\":\"Flue reasoning stream instrumentation\",\"url\":\"https://example.test/flue/reasoning-streams\"}]}\n\n[Assistant tool calls]: summarize_source(url=\"https://example.test/flue/reasoning-streams\")\n\n[Tool result]: {\"summary\":\"Flue emits reasoning, tool execution, and LLM turn events separately.\",\"url\":\"https://example.test/flue/reasoning-streams\"}\n\n\nThis is the PREFIX of a turn that was too large to keep. The SUFFIX (recent work) is retained.\n\nSummarize the prefix to provide context for the retained suffix:\n\n## Original Request\n[What did the user ask for in this turn?]\n\n## Early Progress\n- [Key decisions and work done in the prefix]\n\n## Context for Suffix\n- [Information needed to understand the retained recent work]\n\nBe concise. Focus on what's needed to understand the kept suffix.", -│ "type": "text" -│ } -│ ], -│ "role": "user" -│ } -│ ] -│ output: { -│ "content": [ -│ { -│ "text": "## Original Request\nThe user requested a step-by-step execution of an instrumented research flow involving tool calls to gather information on \"flue instrumentation\" and \"", -│ "textSignature": "{\"v\":1,\"id\":\"msg_04d28ba03840bb65016a705b7358a081a38e69464057840036\"}", -│ "type": "text" -│ } -│ ], -│ "role": "assistant" -│ } -│ metadata: { -│ "flue.api": "openai-responses", -│ "flue.model": "gpt-4o-mini", -│ "flue.provider": "openai", -│ "flue.session": "main", -│ "flue.stop_reason": "stop", -│ "flue.turn_purpose": "compaction_prefix", -│ "model": "gpt-4o-mini", -│ "provider": "openai" -│ } -│ metrics: { -│ "completion_tokens": 0, -│ "duration_ms": 0, -│ "estimated_cost": 0, -│ "prompt_cache_creation_tokens": 0, -│ "prompt_cached_tokens": 0, -│ "prompt_tokens": 0, -│ "tokens": 0 -│ } -└── flue.prompt [task] - input: [ - { - "content": [ - { - "text": "Complete this instrumented research flow. Call exactly one tool per turn and wait for each tool result before choosing the next tool. Step 1: call lookup with query \"flue instrumentation\". Step 2: use the lookup result id as lookupId and call web_search with query \"Braintrust Flue reasoning stream instrumentation\". Step 3: use the first web_search result url and call summarize_source. After summarize_source returns, reply with exactly PROMPT_DONE and no other text.", - "type": "text" - } - ], - "role": "user" - } - ] - output: "PROMPT_DONE" - metadata: { - "flue.operation": "prompt", - "flue.session": "main", - "flue.workflow_name": "instrumentation", - "provider": "flue", - "scenario": "flue-instrumentation" - } - metrics: { - "duration_ms": 0 - } - ├── flue.turn [llm] - │ input: [ - │ { - │ "content": [ - │ { - │ "text": "Complete this instrumented research flow. Call exactly one tool per turn and wait for each tool result before choosing the next tool. Step 1: call lookup with query \"flue instrumentation\". Step 2: use the lookup result id as lookupId and call web_search with query \"Braintrust Flue reasoning stream instrumentation\". Step 3: use the first web_search result url and call summarize_source. After summarize_source returns, reply with exactly PROMPT_DONE and no other text.", - │ "type": "text" - │ } - │ ], - │ "role": "user" - │ } - │ ] - │ output: { - │ "content": [ - │ { - │ "arguments": { - │ "query": "flue instrumentation" - │ }, - │ "id": "", - │ "name": "lookup", - │ "type": "toolCall" - │ } - │ ], - │ "role": "assistant" - │ } - │ metadata: { - │ "flue.api": "openai-responses", - │ "flue.input_mode": "full", - │ "flue.model": "gpt-5.4-nano", - │ "flue.provider": "openai", - │ "flue.session": "main", - │ "flue.stop_reason": "toolUse", - │ "flue.turn_purpose": "agent", - │ "model": "gpt-5.4-nano", - │ "provider": "openai" - │ } - │ metrics: { - │ "completion_tokens": 19, - │ "duration_ms": 0, - │ "estimated_cost": 0.00022475000000000001, - │ "prompt_cache_creation_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 1005, - │ "tokens": 1024 - │ } - ├── tool:lookup [tool] - │ input: { - │ "query": "flue instrumentation" - │ } - │ output: { - │ "content": [ - │ { - │ "text": "{\"id\":\"flue-session-2026\",\"query\":\"flue instrumentation\",\"topic\":\"session instrumentation\"}", - │ "type": "text" - │ } - │ ], - │ "details": { - │ "customTool": "lookup" - │ } - │ } - │ metadata: { - │ "flue.session": "main", - │ "flue.tool_name": "lookup", - │ "provider": "flue" - │ } - │ metrics: { - │ "duration_ms": 0 - │ } - ├── flue.turn [llm] - │ input: [ - │ { - │ "content": [ - │ { - │ "arguments": { - │ "query": "flue instrumentation" - │ }, - │ "id": "", - │ "name": "lookup", - │ "type": "toolCall" - │ } - │ ], - │ "role": "assistant" - │ }, - │ { - │ "content": [ - │ { - │ "text": "{\"id\":\"flue-session-2026\",\"query\":\"flue instrumentation\",\"topic\":\"session instrumentation\"}", - │ "type": "text" - │ } - │ ], - │ "isError": false, - │ "role": "toolResult", - │ "toolCallId": "", - │ "toolName": "lookup" - │ } - │ ] - │ output: { - │ "content": [ - │ { - │ "arguments": { - │ "lookupId": "flue-session-2026", - │ "query": "Braintrust Flue reasoning stream instrumentation" - │ }, - │ "id": "", - │ "name": "web_search", - │ "type": "toolCall" - │ } - │ ], - │ "role": "assistant" - │ } - │ metadata: { - │ "flue.api": "openai-responses", - │ "flue.input_message_offset": 1, - │ "flue.input_mode": "delta", - │ "flue.model": "gpt-5.4-nano", - │ "flue.provider": "openai", - │ "flue.session": "main", - │ "flue.stop_reason": "toolUse", - │ "flue.turn_purpose": "agent", - │ "model": "gpt-5.4-nano", - │ "provider": "openai" - │ } - │ metrics: { - │ "completion_tokens": 34, - │ "duration_ms": 0, - │ "estimated_cost": 0.00025350000000000004, - │ "prompt_cache_creation_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 1055, - │ "tokens": 1089 - │ } - ├── tool:web_search [tool] - │ input: { - │ "lookupId": "flue-session-2026", - │ "query": "Braintrust Flue reasoning stream instrumentation" - │ } - │ output: { - │ "content": [ - │ { - │ "text": "{\"lookupId\":\"flue-session-2026\",\"query\":\"Braintrust Flue reasoning stream instrumentation\",\"results\":[{\"title\":\"Flue reasoning stream instrumentation\",\"url\":\"https://example.test/flue/reasoning-streams\"}]}", - │ "type": "text" - │ } - │ ], - │ "details": { - │ "customTool": "web_search" - │ } - │ } - │ metadata: { - │ "flue.session": "main", - │ "flue.tool_name": "web_search", - │ "provider": "flue" - │ } - │ metrics: { - │ "duration_ms": 0 - │ } - ├── flue.turn [llm] - │ input: [ - │ { - │ "content": [ - │ { - │ "arguments": { - │ "lookupId": "flue-session-2026", - │ "query": "Braintrust Flue reasoning stream instrumentation" - │ }, - │ "id": "", - │ "name": "web_search", - │ "type": "toolCall" - │ } - │ ], - │ "role": "assistant" - │ }, - │ { - │ "content": [ - │ { - │ "text": "{\"lookupId\":\"flue-session-2026\",\"query\":\"Braintrust Flue reasoning stream instrumentation\",\"results\":[{\"title\":\"Flue reasoning stream instrumentation\",\"url\":\"https://example.test/flue/reasoning-streams\"}]}", - │ "type": "text" - │ } - │ ], - │ "isError": false, - │ "role": "toolResult", - │ "toolCallId": "", - │ "toolName": "web_search" - │ } - │ ] - │ output: { - │ "content": [ - │ { - │ "arguments": { - │ "url": "https://example.test/flue/reasoning-streams" - │ }, - │ "id": "", - │ "name": "summarize_source", - │ "type": "toolCall" - │ } - │ ], - │ "role": "assistant" - │ } - │ metadata: { - │ "flue.api": "openai-responses", - │ "flue.input_message_offset": 3, - │ "flue.input_mode": "delta", - │ "flue.model": "gpt-5.4-nano", - │ "flue.provider": "openai", - │ "flue.session": "main", - │ "flue.stop_reason": "toolUse", - │ "flue.turn_purpose": "agent", - │ "model": "gpt-5.4-nano", - │ "provider": "openai" - │ } - │ metrics: { - │ "completion_tokens": 30, - │ "duration_ms": 0, - │ "estimated_cost": 0.00026690000000000004, - │ "prompt_cache_creation_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 1147, - │ "tokens": 1177 - │ } - ├── tool:summarize_source [tool] - │ input: { - │ "url": "https://example.test/flue/reasoning-streams" - │ } - │ output: { - │ "content": [ - │ { - │ "text": "{\"summary\":\"Flue emits reasoning, tool execution, and LLM turn events separately.\",\"url\":\"https://example.test/flue/reasoning-streams\"}", - │ "type": "text" - │ } - │ ], - │ "details": { - │ "customTool": "summarize_source" - │ } - │ } - │ metadata: { - │ "flue.session": "main", - │ "flue.tool_name": "summarize_source", - │ "provider": "flue" - │ } - │ metrics: { - │ "duration_ms": 0 - │ } - └── flue.turn [llm] - input: [ - { - "content": [ - { - "arguments": { - "url": "https://example.test/flue/reasoning-streams" - }, - "id": "", - "name": "summarize_source", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "{\"summary\":\"Flue emits reasoning, tool execution, and LLM turn events separately.\",\"url\":\"https://example.test/flue/reasoning-streams\"}", - "type": "text" - } - ], - "isError": false, - "role": "toolResult", - "toolCallId": "", - "toolName": "summarize_source" - } - ] - output: { - "content": [ - { - "text": "PROMPT_DONE", - "textSignature": "{\"v\":1,\"id\":\"msg_045458052dc10379016a705b62dbd8819189b8287061debf12\",\"phase\":\"final_answer\"}", - "type": "text" - } - ], - "role": "assistant" - } - metadata: { - "flue.api": "openai-responses", - "flue.input_message_offset": 5, - "flue.input_mode": "delta", - "flue.model": "gpt-5.4-nano", - "flue.provider": "openai", - "flue.session": "main", - "flue.stop_reason": "stop", - "flue.turn_purpose": "agent", - "model": "gpt-5.4-nano", - "provider": "openai" - } - metrics: { - "completion_tokens": 7, - "duration_ms": 0, - "estimated_cost": 0.00025315000000000005, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 1222, - "tokens": 1229 - } diff --git a/e2e/scenarios/flue-instrumentation/__snapshots__/flue-v1-0-0-beta-3-explicit.span-tree.json b/e2e/scenarios/flue-instrumentation/__snapshots__/flue-v1-0-0-beta-3-explicit.span-tree.json index 464358cef..aa0fef2c7 100644 --- a/e2e/scenarios/flue-instrumentation/__snapshots__/flue-v1-0-0-beta-3-explicit.span-tree.json +++ b/e2e/scenarios/flue-instrumentation/__snapshots__/flue-v1-0-0-beta-3-explicit.span-tree.json @@ -75,7 +75,7 @@ "input": { "name": "e2e-flue-skill" }, - "output": "Run the skill named \"e2e-flue-skill\".\n\n\nReturn the marker from the provided args exactly once. Output no other text.\n\n\nSupporting skill resources are available relative to this workspace skill directory but are not loaded into context unless needed:\n\n- Base directory: /e2e/scenarios/flue-v1/.agents/skills/e2e-flue-skill\n- Resolve relative resource paths from this directory and read only the files you need.\n", + "output": "Run the skill named \"e2e-flue-skill\".\n\n\nReturn the marker from the provided args exactly once. Output no other text.\n\n\nSupporting skill resources are available relative to this workspace skill directory but are not loaded into context unless needed:\n\n- Base directory: /e2e/scenarios/flue-instrumentation/.agents/skills/e2e-flue-skill\n- Resolve relative resource paths from this directory and read only the files you need.\n", "metadata": { "flue.session": "skill", "flue.tool_name": "activate_skill", @@ -119,7 +119,7 @@ { "content": [ { - "text": "Run the skill named \"e2e-flue-skill\".\n\n\nReturn the marker from the provided args exactly once. Output no other text.\n\n\nSupporting skill resources are available relative to this workspace skill directory but are not loaded into context unless needed:\n\n- Base directory: /e2e/scenarios/flue-v1/.agents/skills/e2e-flue-skill\n- Resolve relative resource paths from this directory and read only the files you need.\n", + "text": "Run the skill named \"e2e-flue-skill\".\n\n\nReturn the marker from the provided args exactly once. Output no other text.\n\n\nSupporting skill resources are available relative to this workspace skill directory but are not loaded into context unless needed:\n\n- Base directory: /e2e/scenarios/flue-instrumentation/.agents/skills/e2e-flue-skill\n- Resolve relative resource paths from this directory and read only the files you need.\n", "type": "text" } ], diff --git a/e2e/scenarios/flue-instrumentation/__snapshots__/flue-v1-0-0-beta-3-explicit.span-tree.txt b/e2e/scenarios/flue-instrumentation/__snapshots__/flue-v1-0-0-beta-3-explicit.span-tree.txt index 104f75f29..60b2ac825 100644 --- a/e2e/scenarios/flue-instrumentation/__snapshots__/flue-v1-0-0-beta-3-explicit.span-tree.txt +++ b/e2e/scenarios/flue-instrumentation/__snapshots__/flue-v1-0-0-beta-3-explicit.span-tree.txt @@ -99,7 +99,7 @@ span_tree: │ │ │ input: { │ │ │ "name": "e2e-flue-skill" │ │ │ } -│ │ │ output: "Run the skill named \"e2e-flue-skill\".\n\n\nReturn the marker from the provided args exactly once. Output no other text.\n\n\nSupporting skill resources are available relative to this workspace skill directory but are not loaded into context unless needed:\n\n- Base directory: /e2e/scenarios/flue-v1/.agents/skills/e2e-flue-skill\n- Resolve relative resource paths from this directory and read only the files you need.\n" +│ │ │ output: "Run the skill named \"e2e-flue-skill\".\n\n\nReturn the marker from the provided args exactly once. Output no other text.\n\n\nSupporting skill resources are available relative to this workspace skill directory but are not loaded into context unless needed:\n\n- Base directory: /e2e/scenarios/flue-instrumentation/.agents/skills/e2e-flue-skill\n- Resolve relative resource paths from this directory and read only the files you need.\n" │ │ │ metadata: { │ │ │ "flue.session": "skill", │ │ │ "flue.tool_name": "activate_skill", @@ -139,7 +139,7 @@ span_tree: │ │ { │ │ "content": [ │ │ { -│ │ "text": "Run the skill named \"e2e-flue-skill\".\n\n\nReturn the marker from the provided args exactly once. Output no other text.\n\n\nSupporting skill resources are available relative to this workspace skill directory but are not loaded into context unless needed:\n\n- Base directory: /e2e/scenarios/flue-v1/.agents/skills/e2e-flue-skill\n- Resolve relative resource paths from this directory and read only the files you need.\n", +│ │ "text": "Run the skill named \"e2e-flue-skill\".\n\n\nReturn the marker from the provided args exactly once. Output no other text.\n\n\nSupporting skill resources are available relative to this workspace skill directory but are not loaded into context unless needed:\n\n- Base directory: /e2e/scenarios/flue-instrumentation/.agents/skills/e2e-flue-skill\n- Resolve relative resource paths from this directory and read only the files you need.\n", │ │ "type": "text" │ │ } │ │ ], diff --git a/e2e/scenarios/flue-instrumentation/__snapshots__/flue-v1-latest-explicit.span-tree.json b/e2e/scenarios/flue-instrumentation/__snapshots__/flue-v1-latest-explicit.span-tree.json index ac0646db1..b4336fbab 100644 --- a/e2e/scenarios/flue-instrumentation/__snapshots__/flue-v1-latest-explicit.span-tree.json +++ b/e2e/scenarios/flue-instrumentation/__snapshots__/flue-v1-latest-explicit.span-tree.json @@ -71,7 +71,7 @@ "input": { "name": "e2e-flue-skill" }, - "output": "Run the skill named \"e2e-flue-skill\".\n\n\nReturn the marker from the provided args exactly once. Output no other text.\n\n\nSupporting skill resources are available relative to this workspace skill directory but are not loaded into context unless needed:\n\n- Base directory: /e2e/scenarios/flue-v1/.agents/skills/e2e-flue-skill\n- Resolve relative resource paths from this directory and read only the files you need.\n", + "output": "Run the skill named \"e2e-flue-skill\".\n\n\nReturn the marker from the provided args exactly once. Output no other text.\n\n\nSupporting skill resources are available relative to this workspace skill directory but are not loaded into context unless needed:\n\n- Base directory: /e2e/scenarios/flue-instrumentation/.agents/skills/e2e-flue-skill\n- Resolve relative resource paths from this directory and read only the files you need.\n", "metadata": { "flue.session": "skill", "flue.tool_name": "activate_skill", @@ -111,7 +111,7 @@ { "content": [ { - "text": "Run the skill named \"e2e-flue-skill\".\n\n\nReturn the marker from the provided args exactly once. Output no other text.\n\n\nSupporting skill resources are available relative to this workspace skill directory but are not loaded into context unless needed:\n\n- Base directory: /e2e/scenarios/flue-v1/.agents/skills/e2e-flue-skill\n- Resolve relative resource paths from this directory and read only the files you need.\n", + "text": "Run the skill named \"e2e-flue-skill\".\n\n\nReturn the marker from the provided args exactly once. Output no other text.\n\n\nSupporting skill resources are available relative to this workspace skill directory but are not loaded into context unless needed:\n\n- Base directory: /e2e/scenarios/flue-instrumentation/.agents/skills/e2e-flue-skill\n- Resolve relative resource paths from this directory and read only the files you need.\n", "type": "text" } ], diff --git a/e2e/scenarios/flue-instrumentation/__snapshots__/flue-v1-latest-explicit.span-tree.txt b/e2e/scenarios/flue-instrumentation/__snapshots__/flue-v1-latest-explicit.span-tree.txt index 2a17573af..dd492f42e 100644 --- a/e2e/scenarios/flue-instrumentation/__snapshots__/flue-v1-latest-explicit.span-tree.txt +++ b/e2e/scenarios/flue-instrumentation/__snapshots__/flue-v1-latest-explicit.span-tree.txt @@ -95,7 +95,7 @@ span_tree: │ │ │ input: { │ │ │ "name": "e2e-flue-skill" │ │ │ } -│ │ │ output: "Run the skill named \"e2e-flue-skill\".\n\n\nReturn the marker from the provided args exactly once. Output no other text.\n\n\nSupporting skill resources are available relative to this workspace skill directory but are not loaded into context unless needed:\n\n- Base directory: /e2e/scenarios/flue-v1/.agents/skills/e2e-flue-skill\n- Resolve relative resource paths from this directory and read only the files you need.\n" +│ │ │ output: "Run the skill named \"e2e-flue-skill\".\n\n\nReturn the marker from the provided args exactly once. Output no other text.\n\n\nSupporting skill resources are available relative to this workspace skill directory but are not loaded into context unless needed:\n\n- Base directory: /e2e/scenarios/flue-instrumentation/.agents/skills/e2e-flue-skill\n- Resolve relative resource paths from this directory and read only the files you need.\n" │ │ │ metadata: { │ │ │ "flue.session": "skill", │ │ │ "flue.tool_name": "activate_skill", @@ -131,7 +131,7 @@ span_tree: │ │ { │ │ "content": [ │ │ { -│ │ "text": "Run the skill named \"e2e-flue-skill\".\n\n\nReturn the marker from the provided args exactly once. Output no other text.\n\n\nSupporting skill resources are available relative to this workspace skill directory but are not loaded into context unless needed:\n\n- Base directory: /e2e/scenarios/flue-v1/.agents/skills/e2e-flue-skill\n- Resolve relative resource paths from this directory and read only the files you need.\n", +│ │ "text": "Run the skill named \"e2e-flue-skill\".\n\n\nReturn the marker from the provided args exactly once. Output no other text.\n\n\nSupporting skill resources are available relative to this workspace skill directory but are not loaded into context unless needed:\n\n- Base directory: /e2e/scenarios/flue-instrumentation/.agents/skills/e2e-flue-skill\n- Resolve relative resource paths from this directory and read only the files you need.\n", │ │ "type": "text" │ │ } │ │ ], diff --git a/e2e/scenarios/flue-instrumentation/assertions.ts b/e2e/scenarios/flue-instrumentation/assertions.ts index 53a597fbd..89b564836 100644 --- a/e2e/scenarios/flue-instrumentation/assertions.ts +++ b/e2e/scenarios/flue-instrumentation/assertions.ts @@ -20,14 +20,6 @@ import { import { SCENARIO_NAME } from "./constants.mjs"; type RunFlueScenario = (harness: { - runNodeScenarioDir: (options: { - entry: string; - env?: Record; - nodeArgs?: string[]; - runContext?: ScenarioRunContext; - scenarioDir: string; - timeoutMs: number; - }) => Promise; runScenarioDir: (options: { entry: string; env?: Record; @@ -155,40 +147,24 @@ function firstSpanIndex( return index === -1 ? Number.MAX_SAFE_INTEGER : index; } -function findFlueDescendants( - events: CapturedLogEvent[], - flueSpan: CapturedLogEvent | undefined, - predicate: (event: CapturedLogEvent) => boolean, -): CapturedLogEvent[] { - return findMatchingDescendants(events, flueSpan, predicate); -} - -function isFlueChildSpan( - event: CapturedLogEvent, - includeAmbientProbeSpans = true, -): boolean { +function isFlueChildSpan(event: CapturedLogEvent): boolean { return ( event.span.name === "flue.turn" || event.span.name?.startsWith("tool:") === true || event.span.name?.startsWith("task:") === true || event.span.name?.startsWith("compaction:") === true || - (includeAmbientProbeSpans && event.span.name === "flue.toolCurrentProbe") || + event.span.name === "flue.toolCurrentProbe" || event.span.name === "flue.task" ); } -function buildSpanTree( - events: CapturedLogEvent[], - includeAmbientProbeSpans: boolean, -): SpanTreeEntry[] { +function buildSpanTree(events: CapturedLogEvent[]): SpanTreeEntry[] { const workflow = findLatestSpanByPrefix(events, "workflow:"); - const workflowCurrentProbe = includeAmbientProbeSpans - ? findLatestChildSpan( - events, - "flue.workflowCurrentProbe", - workflow?.span.id, - ) - : undefined; + const workflowCurrentProbe = findLatestChildSpan( + events, + "flue.workflowCurrentProbe", + workflow?.span.id, + ); const promptSpan = findFlueOperation(events, "flue.prompt"); const skillSpan = findFlueOperation(events, "flue.skill"); const taskSpan = findFlueOperation(events, "flue.task"); @@ -198,28 +174,19 @@ function buildSpanTree( workflow, workflowCurrentProbe, promptSpan, - ...findFlueDescendants(events, promptSpan, (event) => - isFlueChildSpan(event, includeAmbientProbeSpans), - ), + ...findMatchingDescendants(events, promptSpan, isFlueChildSpan), skillSpan, - ...findFlueDescendants(events, skillSpan, (event) => - isFlueChildSpan(event, includeAmbientProbeSpans), - ), + ...findMatchingDescendants(events, skillSpan, isFlueChildSpan), taskSpan, - ...findFlueDescendants(events, taskSpan, (event) => - isFlueChildSpan(event, includeAmbientProbeSpans), - ), + ...findMatchingDescendants(events, taskSpan, isFlueChildSpan), compactSpan, - ...findFlueDescendants(events, compactSpan, (event) => - isFlueChildSpan(event, includeAmbientProbeSpans), - ), + ...findMatchingDescendants(events, compactSpan, isFlueChildSpan), ].flatMap((event) => event ? [{ event, fields: snapshotFields(event) }] : [], ); } export function defineFlueInstrumentationAssertions(options: { - expectAmbientContext?: boolean; name: string; runScenario: RunFlueScenario; snapshotName: string; @@ -232,7 +199,6 @@ export function defineFlueInstrumentationAssertions(options: { ); const timeoutMs = effectiveScenarioTimeoutMs(options.timeoutMs); const testConfig = { timeout: timeoutMs }; - const expectAmbientContext = options.expectAmbientContext ?? true; describe.sequential(options.name, () => { let events: CapturedLogEvent[] = []; @@ -257,46 +223,42 @@ export function defineFlueInstrumentationAssertions(options: { }); }); - if (expectAmbientContext) { - test( - "makes the Flue workflow span current for app spans", - testConfig, - () => { - const workflow = findLatestSpanByPrefix(events, "workflow:"); - const appSpan = findLatestChildSpan( - events, - "flue.workflowCurrentProbe", - workflow?.span.id, - ); - - expect(appSpan).toBeDefined(); - expect(appSpan?.span.parentIds).toEqual([workflow?.span.id]); - expect(appSpan?.output).toBe("active"); - }, - ); - - test("makes Flue tool spans current for app spans", testConfig, () => { + test( + "makes the Flue workflow span current for app spans", + testConfig, + () => { const workflow = findLatestSpanByPrefix(events, "workflow:"); - const promptSpan = findFlueOperation(events, "flue.prompt"); - const lookupToolSpan = findFlueDescendants( - events, - promptSpan, - (event) => event.span.name === "tool:lookup", - )[0]; const appSpan = findLatestChildSpan( events, - "flue.toolCurrentProbe", - lookupToolSpan?.span.id, + "flue.workflowCurrentProbe", + workflow?.span.id, ); expect(appSpan).toBeDefined(); - expect(appSpan?.span.parentIds).toEqual([lookupToolSpan?.span.id]); - expect(appSpan?.output).toBe("lookup-active"); - expect(workflow).toBeDefined(); - }); - } + expect(appSpan?.span.parentIds).toEqual([workflow?.span.id]); + expect(appSpan?.output).toBe("active"); + }, + ); + + test("makes Flue tool spans current for app spans", testConfig, () => { + const promptSpan = findFlueOperation(events, "flue.prompt"); + const lookupToolSpan = findMatchingDescendants( + events, + promptSpan, + (event) => event.span.name === "tool:lookup", + )[0]; + const appSpan = findLatestChildSpan( + events, + "flue.toolCurrentProbe", + lookupToolSpan?.span.id, + ); - test("captures observe-based Flue operation spans", testConfig, () => { + expect(appSpan).toBeDefined(); + expect(appSpan?.span.parentIds).toEqual([lookupToolSpan?.span.id]); + expect(appSpan?.output).toBe("lookup-active"); + }); + + test("captures Flue operation spans", testConfig, () => { for (const flueSpanName of [ "flue.prompt", "flue.skill", @@ -334,10 +296,10 @@ export function defineFlueInstrumentationAssertions(options: { testConfig, () => { const promptSpan = findFlueOperation(events, "flue.prompt"); - const promptChildren = findFlueDescendants( + const promptChildren = findMatchingDescendants( events, promptSpan, - (event) => isFlueChildSpan(event, expectAmbientContext), + isFlueChildSpan, ); const promptTurns = promptChildren.filter( (event) => event.span.name === "flue.turn", @@ -349,7 +311,7 @@ export function defineFlueInstrumentationAssertions(options: { const compactSpan = findFlueOperation(events, "flue.compact"); const allLlmSpans = [promptSpan, skillSpan, compactSpan].flatMap( (span) => - findFlueDescendants( + findMatchingDescendants( events, span, (event) => event.span.name === "flue.turn", @@ -359,12 +321,12 @@ export function defineFlueInstrumentationAssertions(options: { (event) => event.span.name === "tool:lookup", ); const taskSpan = findFlueOperation(events, "flue.task"); - const nestedTaskSpans = findFlueDescendants( + const nestedTaskSpans = findMatchingDescendants( events, taskSpan, (event) => event.span.name === "flue.task", ); - const compaction = findFlueDescendants( + const compaction = findMatchingDescendants( events, compactSpan, (event) => event.span.name?.startsWith("compaction:") === true, @@ -414,15 +376,11 @@ export function defineFlueInstrumentationAssertions(options: { }); test("matches the span tree snapshot", testConfig, async () => { - await matchSpanTreeSnapshot( - buildSpanTree(events, expectAmbientContext), - snapshotPath, - { - normalize: { - additionalProviderIdKeys: ["messageId"], - }, + await matchSpanTreeSnapshot(buildSpanTree(events), snapshotPath, { + normalize: { + additionalProviderIdKeys: ["messageId"], }, - ); + }); }); }); } diff --git a/e2e/scenarios/flue-instrumentation/package.json b/e2e/scenarios/flue-instrumentation/package.json index 456dcc68f..5a98aeb76 100644 --- a/e2e/scenarios/flue-instrumentation/package.json +++ b/e2e/scenarios/flue-instrumentation/package.json @@ -5,19 +5,11 @@ "braintrustScenario": { "bump": { "dependencies": { - "flue-cli-v0-8-latest": { - "package": "@flue/cli", - "range": ">=0.8.0 <0.9.0" - }, "flue-cli-v1-latest": { "package": "@flue/cli", "range": ">=1.0.0-beta.3 <2.0.0-0", "allowPrerelease": true }, - "flue-runtime-v0-8-latest": { - "package": "@flue/runtime", - "range": ">=0.8.0 <0.9.0" - }, "flue-runtime-v1-latest": { "package": "@flue/runtime", "range": ">=1.0.0-beta.3 <2.0.0-0", @@ -31,13 +23,9 @@ } }, "dependencies": { - "@flue/cli": "0.8.0", - "@flue/runtime": "0.8.0", - "flue-cli-v0-8-latest": "npm:@flue/cli@0.8.1", - "flue-cli-v1": "npm:@flue/cli@1.0.0-beta.3", + "@flue/cli": "1.0.0-beta.3", + "@flue/runtime": "1.0.0-beta.3", "flue-cli-v1-latest": "npm:@flue/cli@1.0.0-beta.9", - "flue-runtime-v0-8-latest": "npm:@flue/runtime@0.8.1", - "flue-runtime-v1": "npm:@flue/runtime@1.0.0-beta.3", "flue-runtime-v1-latest": "npm:@flue/runtime@1.0.0-beta.9", "flue-runtime-v2": "npm:@flue/runtime@2.0.0", "flue-runtime-v2-latest": "npm:@flue/runtime@2.0.1", @@ -46,10 +34,6 @@ }, "pnpm": { "overrides": { - "@flue/runtime@0.8.0>@earendil-works/pi-agent-core": "0.75.4", - "@flue/runtime@0.8.0>@earendil-works/pi-ai": "0.75.4", - "@flue/runtime@0.8.1>@earendil-works/pi-agent-core": "0.75.4", - "@flue/runtime@0.8.1>@earendil-works/pi-ai": "0.75.4", "@hono/node-server@>=2.0.0 <2.0.10": "^2.0.10", "fast-uri@<3.1.5": "^3.1.5", "hono@>=4.0.0 <4.12.27": "^4.12.27", diff --git a/e2e/scenarios/flue-instrumentation/pnpm-lock.yaml b/e2e/scenarios/flue-instrumentation/pnpm-lock.yaml index d1b7e728c..d1eec91bd 100644 --- a/e2e/scenarios/flue-instrumentation/pnpm-lock.yaml +++ b/e2e/scenarios/flue-instrumentation/pnpm-lock.yaml @@ -5,10 +5,6 @@ settings: excludeLinksFromLockfile: false overrides: - '@flue/runtime@0.8.0>@earendil-works/pi-agent-core': 0.75.4 - '@flue/runtime@0.8.0>@earendil-works/pi-ai': 0.75.4 - '@flue/runtime@0.8.1>@earendil-works/pi-agent-core': 0.75.4 - '@flue/runtime@0.8.1>@earendil-works/pi-ai': 0.75.4 '@hono/node-server@>=2.0.0 <2.0.10': ^2.0.10 fast-uri@<3.1.5: ^3.1.5 hono@>=4.0.0 <4.12.27: ^4.12.27 @@ -23,38 +19,26 @@ importers: .: dependencies: '@flue/cli': - specifier: 0.8.0 - version: 0.8.0(@standard-schema/spec@1.1.0)(@types/json-schema@7.0.15)(@types/node@25.9.1)(esbuild@0.27.3)(typebox@1.3.7)(workerd@1.20260730.1)(wrangler@4.95.0)(yaml@2.9.0)(zod-to-json-schema@3.25.2(zod@4.4.3))(zod@4.4.3) + specifier: 1.0.0-beta.3 + version: 1.0.0-beta.3(@standard-schema/spec@1.1.0)(@types/json-schema@7.0.15)(@types/node@25.9.1)(esbuild@0.27.3)(hono@4.12.31)(typebox@1.3.7)(wrangler@4.95.0)(ws@8.21.0)(yaml@2.9.0)(zod-to-json-schema@3.25.2(zod@4.4.3))(zod@4.4.3) '@flue/runtime': - specifier: 0.8.0 - version: 0.8.0(@standard-schema/spec@1.1.0)(@types/json-schema@7.0.15)(typebox@1.3.7)(zod-to-json-schema@3.25.2(zod@4.4.3))(zod@4.4.3) - flue-cli-v0-8-latest: - specifier: npm:@flue/cli@0.8.1 - version: '@flue/cli@0.8.1(@standard-schema/spec@1.1.0)(@types/json-schema@7.0.15)(@types/node@25.9.1)(esbuild@0.27.3)(typebox@1.3.7)(workerd@1.20260730.1)(wrangler@4.95.0)(yaml@2.9.0)(zod-to-json-schema@3.25.2(zod@4.4.3))(zod@4.4.3)' - flue-cli-v1: - specifier: npm:@flue/cli@1.0.0-beta.3 - version: '@flue/cli@1.0.0-beta.3(@standard-schema/spec@1.1.0)(@types/json-schema@7.0.15)(@types/node@25.9.1)(esbuild@0.27.3)(hono@4.12.31)(typebox@1.3.7)(wrangler@4.95.0)(ws@8.20.1)(yaml@2.9.0)(zod-to-json-schema@3.25.2(zod@4.4.3))(zod@4.4.3)' + specifier: 1.0.0-beta.3 + version: 1.0.0-beta.3(@standard-schema/spec@1.1.0)(@types/json-schema@7.0.15)(typebox@1.3.7)(ws@8.21.0)(zod-to-json-schema@3.25.2(zod@4.4.3))(zod@4.4.3) flue-cli-v1-latest: specifier: npm:@flue/cli@1.0.0-beta.9 - version: '@flue/cli@1.0.0-beta.9(@standard-schema/spec@1.1.0)(@types/json-schema@7.0.15)(@types/node@25.9.1)(esbuild@0.27.3)(hono@4.12.31)(typebox@1.3.7)(wrangler@4.95.0)(ws@8.20.1)(yaml@2.9.0)(zod-to-json-schema@3.25.2(zod@4.4.3))(zod@4.4.3)' - flue-runtime-v0-8-latest: - specifier: npm:@flue/runtime@0.8.1 - version: '@flue/runtime@0.8.1(@standard-schema/spec@1.1.0)(@types/json-schema@7.0.15)(typebox@1.3.7)(zod-to-json-schema@3.25.2(zod@4.4.3))(zod@4.4.3)' - flue-runtime-v1: - specifier: npm:@flue/runtime@1.0.0-beta.3 - version: '@flue/runtime@1.0.0-beta.3(@standard-schema/spec@1.1.0)(@types/json-schema@7.0.15)(typebox@1.3.7)(ws@8.20.1)(zod-to-json-schema@3.25.2(zod@4.4.3))(zod@4.4.3)' + version: '@flue/cli@1.0.0-beta.9(@standard-schema/spec@1.1.0)(@types/json-schema@7.0.15)(@types/node@25.9.1)(esbuild@0.27.3)(hono@4.12.31)(typebox@1.3.7)(wrangler@4.95.0)(ws@8.21.0)(yaml@2.9.0)(zod-to-json-schema@3.25.2(zod@4.4.3))(zod@4.4.3)' flue-runtime-v1-latest: specifier: npm:@flue/runtime@1.0.0-beta.9 - version: '@flue/runtime@1.0.0-beta.9(@standard-schema/spec@1.1.0)(@types/json-schema@7.0.15)(typebox@1.3.7)(ws@8.20.1)(zod-to-json-schema@3.25.2(zod@4.4.3))(zod@4.4.3)' + version: '@flue/runtime@1.0.0-beta.9(@standard-schema/spec@1.1.0)(@types/json-schema@7.0.15)(typebox@1.3.7)(ws@8.21.0)(zod-to-json-schema@3.25.2(zod@4.4.3))(zod@4.4.3)' flue-runtime-v2: specifier: npm:@flue/runtime@2.0.0 - version: '@flue/runtime@2.0.0(@modelcontextprotocol/sdk@1.29.0(zod@4.4.3))(ws@8.20.1)(zod@4.4.3)' + version: '@flue/runtime@2.0.0(@modelcontextprotocol/sdk@1.29.0(zod@4.4.3))(ws@8.21.0)(zod@4.4.3)' flue-runtime-v2-latest: specifier: npm:@flue/runtime@2.0.1 - version: '@flue/runtime@2.0.1(@modelcontextprotocol/sdk@1.29.0(zod@4.4.3))(ws@8.20.1)(zod@4.4.3)' + version: '@flue/runtime@2.0.1(@modelcontextprotocol/sdk@1.29.0(zod@4.4.3))(ws@8.21.0)(zod@4.4.3)' pi-ai-v2: specifier: npm:@earendil-works/pi-ai@0.83.0 - version: '@earendil-works/pi-ai@0.83.0(@modelcontextprotocol/sdk@1.29.0(zod@4.4.3))(ws@8.20.1)(zod@4.4.3)' + version: '@earendil-works/pi-ai@0.83.0(@modelcontextprotocol/sdk@1.29.0(zod@4.4.3))(ws@8.21.0)(zod@4.4.3)' valibot: specifier: 1.4.2 version: 1.4.2 @@ -191,12 +175,6 @@ packages: workerd: optional: true - '@cloudflare/vite-plugin@1.39.0': - resolution: {integrity: sha512-AHC+KSR+3dtGu7Ab7I0Ode4Whx12TxMEmiZt7w+Fc3/2wYNByIzbb6cndWZ78tnveFdO1xhNLv1YaNngxGtOPg==} - peerDependencies: - vite: ^6.1.0 || ^7.0.0 || ^8.0.0 - wrangler: ^4.95.0 - '@cloudflare/vite-plugin@1.50.0': resolution: {integrity: sha512-zIhZim7Kr7OC22rlOkU81BLj0oRlUOqPRX+E6RU3hyRYg4xU1MbA3yiMIwFONr+jc/uV7Fxs20NlXrB89Fqjqg==} hasBin: true @@ -273,10 +251,6 @@ packages: engines: {node: '>=18.0.0'} hasBin: true - '@earendil-works/pi-agent-core@0.75.4': - resolution: {integrity: sha512-cGYbysb4EqUf0B28OeqFq2ppm1XF3bYBOP71q9dv38yf/UJfzMjiXBeNelrcio+QWIoVrW+xzYm7sMzYIUc9Og==} - engines: {node: '>=22.19.0'} - '@earendil-works/pi-agent-core@0.79.10': resolution: {integrity: sha512-XKxgdjhcPuyjrthCOFSgfzT3xZ1uBrJ1IMVDxci1to6hIN6BIg9J5iY8q0pGXK1DLgATLP23da+1UyZLwA360Q==} engines: {node: '>=22.19.0'} @@ -289,11 +263,6 @@ packages: resolution: {integrity: sha512-RorGp9OH5l3ElpuC5a5ZQ2eWcchZGXflXRzVGkV99y3y6tT+LLNyxoYIdVKvTKWEObwhExeQbTH0fI2tE4iX4g==} engines: {node: '>=22.19.0'} - '@earendil-works/pi-ai@0.75.4': - resolution: {integrity: sha512-m/w8Hh3vQ0rAycwJiJWdzkypkn4295f4eq/966lDRy8aX5sk6bgYXH8TQmL16TO7Uwc7MbJG0QoyFHgX8RqXUQ==} - engines: {node: '>=22.19.0'} - hasBin: true - '@earendil-works/pi-ai@0.79.10': resolution: {integrity: sha512-9jR23tOl0BIUdQMn70Gr72xYBpM7Xgl9Lyv7gAnU1USfkNRuYG/f/edLl+n/Dp/RafDW3JI4DF7y/GhgkORuew==} engines: {node: '>=22.19.0'} @@ -477,26 +446,6 @@ packages: cpu: [x64] os: [win32] - '@flue/cli@0.8.0': - resolution: {integrity: sha512-iWTVoSCZ0/D5T5O4E+dha9IT7yx1/qqjsEGLtedd/wnlp2nK45qTrjutBS2pgenGNzdQ6wlPzMPkP8MA9z49yw==} - engines: {node: '>=22.18.0'} - hasBin: true - peerDependencies: - wrangler: ^4.94.0 - peerDependenciesMeta: - wrangler: - optional: true - - '@flue/cli@0.8.1': - resolution: {integrity: sha512-8aeaSf7RsXj4A4P+eQgTj8l2+17NScrVCK3UcbMuQ2pwAu84v5t6SQR2XvYuqActlidTIFR6+xxWS7dR9T7yZw==} - engines: {node: '>=22.18.0'} - hasBin: true - peerDependencies: - wrangler: ^4.94.0 - peerDependenciesMeta: - wrangler: - optional: true - '@flue/cli@1.0.0-beta.3': resolution: {integrity: sha512-i8Lo0koB0gcy3bbCp+kwA1C522PsxvN8ydexDwR3rMkWisitG3FVmqmsLPkWBetspmXBLDbyDq39+AdSoXf4EQ==} engines: {node: '>=22.19.0'} @@ -507,14 +456,6 @@ packages: engines: {node: '>=22.19.0'} hasBin: true - '@flue/runtime@0.8.0': - resolution: {integrity: sha512-GCXJOZtb6s+HuMBzB4I41uQzxX7pELfdorD3gnWnkYoNFjHW8D/ncjXMTsczyl2tkP1IPK0C1Ffw0zwGqkmO7g==} - engines: {node: '>=22.18.0'} - - '@flue/runtime@0.8.1': - resolution: {integrity: sha512-nhIiNLr4NmsK6xgYgFt+mTFTKhHxMn++4gjzygYQIUqQK0GR4hgIjpvosl/361Xx087+8N0wNWh2MEVStoZCIg==} - engines: {node: '>=22.18.0'} - '@flue/runtime@1.0.0-beta.3': resolution: {integrity: sha512-VkWz6lRaS73fu6Y8Wh+CjAHjrLmICUmDwURQCA7dtXWP2O99UyvDQ/5Cbg9DYpsYZpaFuOhWzOCj+Igc4F/LbA==} engines: {node: '>=22.19.0'} @@ -903,9 +844,6 @@ packages: '@microsoft/fetch-event-source@2.0.1': resolution: {integrity: sha512-W6CLUJ2eBMw3Rec70qrsEW0jOm/3twwJv21mrmj2yORiaVmVYGS4sSS5yUwvQc1ZlDLYGPnClVWmUUMagKNsfA==} - '@mistralai/mistralai@2.2.1': - resolution: {integrity: sha512-uKU8CZmL2RzYKmplsU01hii4p3pe4HqJefpWNRWXm1Tcm0Sm4xXfwSLIy4k7ZCPlbETCGcp69E7hZs+WOJ5itQ==} - '@mistralai/mistralai@2.2.6': resolution: {integrity: sha512-W8pX7zHxjJvMIpw8JMxeJEleapXX0Q9NPszdNzqkM3MIEoIGPObdodujj+WHteXEvGfaP/AMwlNyRfEzSY6dQQ==} peerDependencies: @@ -2002,11 +1940,6 @@ packages: resolution: {integrity: sha512-SMguiTnYrhpLdk3PwfzHeotrcwi8bNV4iemL9tx9poR/yeaMYwB9VzR1w7b57DuWpuqR8n6oZboi0hj3AxZxQg==} hasBin: true - semver@7.8.0: - resolution: {integrity: sha512-AcM7dV/5ul4EekoQ29Agm5vri8JNqRyj39o0qpX6vDF2GZrtutZl5RwgD1XnZjiTAfncsJhMI48QQH3sN87YNA==} - engines: {node: '>=10'} - hasBin: true - semver@7.8.5: resolution: {integrity: sha512-Y7/KDsb8LjooZpwaqGyulO6DQlksgCncchHGk+sZIY4SBvUocMBEFH5Ur1fI4dV+Jvl0w6cjvucaIi40puRioA==} engines: {node: '>=10'} @@ -2154,10 +2087,6 @@ packages: resolution: {integrity: sha512-IDxfleLmmbSskfWSUATiN1nfn2rDuvnMOqb5CWR92iIfojA0Ud+ulOAAEQ57LPr9rWmsreUyf5lwyao+7GNNVw==} engines: {node: '>=20.18.1'} - undici@7.28.0: - resolution: {integrity: sha512-cRZYrTDwWznlnRiPjggAGxZXanty6M8RV1ff8Wm4LWXBp7/IG8v5DnOm74DtUBp9OONpK75YlPnIjQqX0dBDtA==} - engines: {node: '>=20.18.1'} - unenv@2.0.0-rc.24: resolution: {integrity: sha512-i7qRCmY42zmCwnYlh9H2SvLEypEFGye5iRmEMKjcGi7zk9UquigRjFtTLz0TYqr0ZGLZhaMHl/foy1bZR+Cwlw==} @@ -2553,19 +2482,6 @@ snapshots: optionalDependencies: workerd: 1.20260730.1 - '@cloudflare/vite-plugin@1.39.0(vite@8.0.14(@types/node@25.9.1)(esbuild@0.27.3)(yaml@2.9.0))(workerd@1.20260730.1)(wrangler@4.95.0)': - dependencies: - '@cloudflare/unenv-preset': 2.16.1(unenv@2.0.0-rc.24)(workerd@1.20260730.1) - miniflare: 4.20260526.0 - unenv: 2.0.0-rc.24 - vite: 8.0.14(@types/node@25.9.1)(esbuild@0.27.3)(yaml@2.9.0) - wrangler: 4.95.0 - ws: 8.20.1 - transitivePeerDependencies: - - bufferutil - - utf-8-validate - - workerd - '@cloudflare/vite-plugin@1.50.0(vite@8.0.14(@types/node@25.9.1)(esbuild@0.27.3)(yaml@2.9.0))(wrangler@4.95.0)': dependencies: '@cloudflare/unenv-preset': 2.16.1(unenv@2.0.0-rc.24)(workerd@1.20260730.1) @@ -2618,9 +2534,9 @@ snapshots: '@microsoft/fetch-event-source': 2.0.1 fastq: 1.20.1 - '@earendil-works/pi-agent-core@0.75.4(@modelcontextprotocol/sdk@1.29.0(zod@4.4.3))(ws@8.20.1)(zod@4.4.3)': + '@earendil-works/pi-agent-core@0.79.10(@modelcontextprotocol/sdk@1.29.0(zod@4.4.3))(ws@8.21.0)(zod@4.4.3)': dependencies: - '@earendil-works/pi-ai': 0.75.4(@modelcontextprotocol/sdk@1.29.0(zod@4.4.3))(ws@8.20.1)(zod@4.4.3) + '@earendil-works/pi-ai': 0.79.10(@modelcontextprotocol/sdk@1.29.0(zod@4.4.3))(ws@8.21.0)(zod@4.4.3) ignore: 7.0.5 typebox: 1.1.38 yaml: 2.9.0 @@ -2632,9 +2548,9 @@ snapshots: - ws - zod - '@earendil-works/pi-agent-core@0.79.10(@modelcontextprotocol/sdk@1.29.0(zod@4.4.3))(ws@8.20.1)(zod@4.4.3)': + '@earendil-works/pi-agent-core@0.80.10(@modelcontextprotocol/sdk@1.29.0(zod@4.4.3))(ws@8.21.0)(zod@4.4.3)': dependencies: - '@earendil-works/pi-ai': 0.79.10(@modelcontextprotocol/sdk@1.29.0(zod@4.4.3))(ws@8.20.1)(zod@4.4.3) + '@earendil-works/pi-ai': 0.80.10(@modelcontextprotocol/sdk@1.29.0(zod@4.4.3))(ws@8.21.0)(zod@4.4.3) ignore: 7.0.5 typebox: 1.1.38 yaml: 2.9.0 @@ -2646,23 +2562,9 @@ snapshots: - ws - zod - '@earendil-works/pi-agent-core@0.80.10(@modelcontextprotocol/sdk@1.29.0(zod@4.4.3))(ws@8.20.1)(zod@4.4.3)': + '@earendil-works/pi-agent-core@0.83.0(@modelcontextprotocol/sdk@1.29.0(zod@4.4.3))(ws@8.21.0)(zod@4.4.3)': dependencies: - '@earendil-works/pi-ai': 0.80.10(@modelcontextprotocol/sdk@1.29.0(zod@4.4.3))(ws@8.20.1)(zod@4.4.3) - ignore: 7.0.5 - typebox: 1.1.38 - yaml: 2.9.0 - transitivePeerDependencies: - - '@modelcontextprotocol/sdk' - - bufferutil - - supports-color - - utf-8-validate - - ws - - zod - - '@earendil-works/pi-agent-core@0.83.0(@modelcontextprotocol/sdk@1.29.0(zod@4.4.3))(ws@8.20.1)(zod@4.4.3)': - dependencies: - '@earendil-works/pi-ai': 0.83.0(@modelcontextprotocol/sdk@1.29.0(zod@4.4.3))(ws@8.20.1)(zod@4.4.3) + '@earendil-works/pi-ai': 0.83.0(@modelcontextprotocol/sdk@1.29.0(zod@4.4.3))(ws@8.21.0)(zod@4.4.3) diff: 8.0.4 ignore: 7.0.5 typebox: 1.3.7 @@ -2675,26 +2577,7 @@ snapshots: - ws - zod - '@earendil-works/pi-ai@0.75.4(@modelcontextprotocol/sdk@1.29.0(zod@4.4.3))(ws@8.20.1)(zod@4.4.3)': - dependencies: - '@anthropic-ai/sdk': 0.91.1(zod@4.4.3) - '@aws-sdk/client-bedrock-runtime': 3.1048.0 - '@google/genai': 1.52.0(@modelcontextprotocol/sdk@1.29.0(zod@4.4.3)) - '@mistralai/mistralai': 2.2.1 - http-proxy-agent: 7.0.2 - https-proxy-agent: 7.0.6 - openai: 6.26.0(ws@8.20.1)(zod@4.4.3) - partial-json: 0.1.7 - typebox: 1.1.38 - transitivePeerDependencies: - - '@modelcontextprotocol/sdk' - - bufferutil - - supports-color - - utf-8-validate - - ws - - zod - - '@earendil-works/pi-ai@0.79.10(@modelcontextprotocol/sdk@1.29.0(zod@4.4.3))(ws@8.20.1)(zod@4.4.3)': + '@earendil-works/pi-ai@0.79.10(@modelcontextprotocol/sdk@1.29.0(zod@4.4.3))(ws@8.21.0)(zod@4.4.3)': dependencies: '@anthropic-ai/sdk': 0.91.1(zod@4.4.3) '@aws-sdk/client-bedrock-runtime': 3.1048.0 @@ -2704,7 +2587,7 @@ snapshots: '@smithy/node-http-handler': 4.7.3 http-proxy-agent: 7.0.2 https-proxy-agent: 7.0.6 - openai: 6.26.0(ws@8.20.1)(zod@4.4.3) + openai: 6.26.0(ws@8.21.0)(zod@4.4.3) partial-json: 0.1.7 typebox: 1.1.38 transitivePeerDependencies: @@ -2715,7 +2598,7 @@ snapshots: - ws - zod - '@earendil-works/pi-ai@0.80.10(@modelcontextprotocol/sdk@1.29.0(zod@4.4.3))(ws@8.20.1)(zod@4.4.3)': + '@earendil-works/pi-ai@0.80.10(@modelcontextprotocol/sdk@1.29.0(zod@4.4.3))(ws@8.21.0)(zod@4.4.3)': dependencies: '@anthropic-ai/sdk': 0.91.1(zod@4.4.3) '@aws-sdk/client-bedrock-runtime': 3.1048.0 @@ -2725,7 +2608,7 @@ snapshots: '@smithy/node-http-handler': 4.7.3 http-proxy-agent: 7.0.2 https-proxy-agent: 7.0.6 - openai: 6.26.0(ws@8.20.1)(zod@4.4.3) + openai: 6.26.0(ws@8.21.0)(zod@4.4.3) partial-json: 0.1.7 typebox: 1.1.38 transitivePeerDependencies: @@ -2736,7 +2619,7 @@ snapshots: - ws - zod - '@earendil-works/pi-ai@0.83.0(@modelcontextprotocol/sdk@1.29.0(zod@4.4.3))(ws@8.20.1)(zod@4.4.3)': + '@earendil-works/pi-ai@0.83.0(@modelcontextprotocol/sdk@1.29.0(zod@4.4.3))(ws@8.21.0)(zod@4.4.3)': dependencies: '@anthropic-ai/sdk': 0.91.1(zod@4.4.3) '@aws-sdk/client-bedrock-runtime': 3.1048.0 @@ -2746,7 +2629,7 @@ snapshots: '@smithy/node-http-handler': 4.7.3 http-proxy-agent: 7.0.2 https-proxy-agent: 7.0.6 - openai: 6.26.0(ws@8.20.1)(zod@4.4.3) + openai: 6.26.0(ws@8.21.0)(zod@4.4.3) partial-json: 0.1.7 typebox: 1.3.7 transitivePeerDependencies: @@ -2856,88 +2739,10 @@ snapshots: '@esbuild/win32-x64@0.27.3': optional: true - '@flue/cli@0.8.0(@standard-schema/spec@1.1.0)(@types/json-schema@7.0.15)(@types/node@25.9.1)(esbuild@0.27.3)(typebox@1.3.7)(workerd@1.20260730.1)(wrangler@4.95.0)(yaml@2.9.0)(zod-to-json-schema@3.25.2(zod@4.4.3))(zod@4.4.3)': - dependencies: - '@cloudflare/vite-plugin': 1.39.0(vite@8.0.14(@types/node@25.9.1)(esbuild@0.27.3)(yaml@2.9.0))(workerd@1.20260730.1)(wrangler@4.95.0) - '@flue/runtime': 0.8.0(@standard-schema/spec@1.1.0)(@types/json-schema@7.0.15)(typebox@1.3.7)(zod-to-json-schema@3.25.2(zod@4.4.3))(zod@4.4.3) - '@vercel/detect-agent': 1.2.3 - package-up: 5.0.0 - valibot: 1.4.2 - vite: 8.0.14(@types/node@25.9.1)(esbuild@0.27.3)(yaml@2.9.0) - optionalDependencies: - wrangler: 4.95.0 - transitivePeerDependencies: - - '@cfworker/json-schema' - - '@standard-schema/spec' - - '@types/json-schema' - - '@types/node' - - '@vitejs/devtools' - - arktype - - bufferutil - - effect - - esbuild - - jiti - - less - - sass - - sass-embedded - - stylus - - sugarss - - supports-color - - sury - - terser - - tsx - - typebox - - typescript - - utf-8-validate - - workerd - - yaml - - zod - - zod-openapi - - zod-to-json-schema - - '@flue/cli@0.8.1(@standard-schema/spec@1.1.0)(@types/json-schema@7.0.15)(@types/node@25.9.1)(esbuild@0.27.3)(typebox@1.3.7)(workerd@1.20260730.1)(wrangler@4.95.0)(yaml@2.9.0)(zod-to-json-schema@3.25.2(zod@4.4.3))(zod@4.4.3)': - dependencies: - '@cloudflare/vite-plugin': 1.39.0(vite@8.0.14(@types/node@25.9.1)(esbuild@0.27.3)(yaml@2.9.0))(workerd@1.20260730.1)(wrangler@4.95.0) - '@flue/runtime': 0.8.1(@standard-schema/spec@1.1.0)(@types/json-schema@7.0.15)(typebox@1.3.7)(zod-to-json-schema@3.25.2(zod@4.4.3))(zod@4.4.3) - '@vercel/detect-agent': 1.2.3 - package-up: 5.0.0 - valibot: 1.4.2 - vite: 8.0.14(@types/node@25.9.1)(esbuild@0.27.3)(yaml@2.9.0) - optionalDependencies: - wrangler: 4.95.0 - transitivePeerDependencies: - - '@cfworker/json-schema' - - '@standard-schema/spec' - - '@types/json-schema' - - '@types/node' - - '@vitejs/devtools' - - arktype - - bufferutil - - effect - - esbuild - - jiti - - less - - sass - - sass-embedded - - stylus - - sugarss - - supports-color - - sury - - terser - - tsx - - typebox - - typescript - - utf-8-validate - - workerd - - yaml - - zod - - zod-openapi - - zod-to-json-schema - - '@flue/cli@1.0.0-beta.3(@standard-schema/spec@1.1.0)(@types/json-schema@7.0.15)(@types/node@25.9.1)(esbuild@0.27.3)(hono@4.12.31)(typebox@1.3.7)(wrangler@4.95.0)(ws@8.20.1)(yaml@2.9.0)(zod-to-json-schema@3.25.2(zod@4.4.3))(zod@4.4.3)': + '@flue/cli@1.0.0-beta.3(@standard-schema/spec@1.1.0)(@types/json-schema@7.0.15)(@types/node@25.9.1)(esbuild@0.27.3)(hono@4.12.31)(typebox@1.3.7)(wrangler@4.95.0)(ws@8.21.0)(yaml@2.9.0)(zod-to-json-schema@3.25.2(zod@4.4.3))(zod@4.4.3)': dependencies: '@cloudflare/vite-plugin': 1.50.0(vite@8.0.14(@types/node@25.9.1)(esbuild@0.27.3)(yaml@2.9.0))(wrangler@4.95.0) - '@flue/runtime': 1.0.0-beta.3(@standard-schema/spec@1.1.0)(@types/json-schema@7.0.15)(typebox@1.3.7)(ws@8.20.1)(zod-to-json-schema@3.25.2(zod@4.4.3))(zod@4.4.3) + '@flue/runtime': 1.0.0-beta.3(@standard-schema/spec@1.1.0)(@types/json-schema@7.0.15)(typebox@1.3.7)(ws@8.21.0)(zod-to-json-schema@3.25.2(zod@4.4.3))(zod@4.4.3) '@flue/sdk': 1.0.0-beta.3 '@hono/node-server': 2.0.11(hono@4.12.31) '@vercel/detect-agent': 1.2.3 @@ -2979,10 +2784,10 @@ snapshots: - zod-openapi - zod-to-json-schema - '@flue/cli@1.0.0-beta.9(@standard-schema/spec@1.1.0)(@types/json-schema@7.0.15)(@types/node@25.9.1)(esbuild@0.27.3)(hono@4.12.31)(typebox@1.3.7)(wrangler@4.95.0)(ws@8.20.1)(yaml@2.9.0)(zod-to-json-schema@3.25.2(zod@4.4.3))(zod@4.4.3)': + '@flue/cli@1.0.0-beta.9(@standard-schema/spec@1.1.0)(@types/json-schema@7.0.15)(@types/node@25.9.1)(esbuild@0.27.3)(hono@4.12.31)(typebox@1.3.7)(wrangler@4.95.0)(ws@8.21.0)(yaml@2.9.0)(zod-to-json-schema@3.25.2(zod@4.4.3))(zod@4.4.3)': dependencies: '@cloudflare/vite-plugin': 1.50.0(vite@8.0.14(@types/node@25.9.1)(esbuild@0.27.3)(yaml@2.9.0))(wrangler@4.95.0) - '@flue/runtime': 1.0.0-beta.9(@standard-schema/spec@1.1.0)(@types/json-schema@7.0.15)(typebox@1.3.7)(ws@8.20.1)(zod-to-json-schema@3.25.2(zod@4.4.3))(zod@4.4.3) + '@flue/runtime': 1.0.0-beta.9(@standard-schema/spec@1.1.0)(@types/json-schema@7.0.15)(typebox@1.3.7)(ws@8.21.0)(zod-to-json-schema@3.25.2(zod@4.4.3))(zod@4.4.3) '@flue/sdk': 1.0.0-beta.9 '@hono/node-server': 2.0.11(hono@4.12.31) '@vercel/detect-agent': 1.2.3 @@ -3024,80 +2829,10 @@ snapshots: - zod-openapi - zod-to-json-schema - '@flue/runtime@0.8.0(@standard-schema/spec@1.1.0)(@types/json-schema@7.0.15)(typebox@1.3.7)(zod-to-json-schema@3.25.2(zod@4.4.3))(zod@4.4.3)': - dependencies: - '@earendil-works/pi-agent-core': 0.75.4(@modelcontextprotocol/sdk@1.29.0(zod@4.4.3))(ws@8.20.1)(zod@4.4.3) - '@earendil-works/pi-ai': 0.75.4(@modelcontextprotocol/sdk@1.29.0(zod@4.4.3))(ws@8.20.1)(zod@4.4.3) - '@hono/node-server': 2.0.11(hono@4.12.31) - '@hono/standard-validator': 0.2.2(@standard-schema/spec@1.1.0)(hono@4.12.31) - '@modelcontextprotocol/sdk': 1.29.0(zod@4.4.3) - '@standard-community/standard-json': 0.3.5(@standard-schema/spec@1.1.0)(@types/json-schema@7.0.15)(@valibot/to-json-schema@1.7.0(valibot@1.4.2))(quansync@0.2.11)(typebox@1.3.7)(valibot@1.4.2)(zod-to-json-schema@3.25.2(zod@4.4.3))(zod@4.4.3) - '@standard-community/standard-openapi': 0.2.9(@standard-community/standard-json@0.3.5(@standard-schema/spec@1.1.0)(@types/json-schema@7.0.15)(@valibot/to-json-schema@1.7.0(valibot@1.4.2))(quansync@0.2.11)(typebox@1.3.7)(valibot@1.4.2)(zod-to-json-schema@3.25.2(zod@4.4.3))(zod@4.4.3))(@standard-schema/spec@1.1.0)(openapi-types@12.1.3)(typebox@1.3.7)(valibot@1.4.2)(zod@4.4.3) - '@valibot/to-json-schema': 1.7.0(valibot@1.4.2) - hono: 4.12.31 - hono-openapi: 1.3.0(@hono/standard-validator@0.2.2(@standard-schema/spec@1.1.0)(hono@4.12.31))(@standard-community/standard-json@0.3.5(@standard-schema/spec@1.1.0)(@types/json-schema@7.0.15)(@valibot/to-json-schema@1.7.0(valibot@1.4.2))(quansync@0.2.11)(typebox@1.3.7)(valibot@1.4.2)(zod-to-json-schema@3.25.2(zod@4.4.3))(zod@4.4.3))(@standard-community/standard-openapi@0.2.9(@standard-community/standard-json@0.3.5(@standard-schema/spec@1.1.0)(@types/json-schema@7.0.15)(@valibot/to-json-schema@1.7.0(valibot@1.4.2))(quansync@0.2.11)(typebox@1.3.7)(valibot@1.4.2)(zod-to-json-schema@3.25.2(zod@4.4.3))(zod@4.4.3))(@standard-schema/spec@1.1.0)(openapi-types@12.1.3)(typebox@1.3.7)(valibot@1.4.2)(zod@4.4.3))(@types/json-schema@7.0.15)(hono@4.12.31)(openapi-types@12.1.3) - js-yaml: 4.1.1 - just-bash: 3.0.1 - openapi-types: 12.1.3 - quansync: 0.2.11 - ulidx: 2.4.1 - valibot: 1.4.2 - ws: 8.20.1 - transitivePeerDependencies: - - '@cfworker/json-schema' - - '@standard-schema/spec' - - '@types/json-schema' - - arktype - - bufferutil - - effect - - supports-color - - sury - - typebox - - typescript - - utf-8-validate - - zod - - zod-openapi - - zod-to-json-schema - - '@flue/runtime@0.8.1(@standard-schema/spec@1.1.0)(@types/json-schema@7.0.15)(typebox@1.3.7)(zod-to-json-schema@3.25.2(zod@4.4.3))(zod@4.4.3)': - dependencies: - '@earendil-works/pi-agent-core': 0.75.4(@modelcontextprotocol/sdk@1.29.0(zod@4.4.3))(ws@8.20.1)(zod@4.4.3) - '@earendil-works/pi-ai': 0.75.4(@modelcontextprotocol/sdk@1.29.0(zod@4.4.3))(ws@8.20.1)(zod@4.4.3) - '@hono/node-server': 2.0.11(hono@4.12.31) - '@hono/standard-validator': 0.2.2(@standard-schema/spec@1.1.0)(hono@4.12.31) - '@modelcontextprotocol/sdk': 1.29.0(zod@4.4.3) - '@standard-community/standard-json': 0.3.5(@standard-schema/spec@1.1.0)(@types/json-schema@7.0.15)(@valibot/to-json-schema@1.7.0(valibot@1.4.2))(quansync@0.2.11)(typebox@1.3.7)(valibot@1.4.2)(zod-to-json-schema@3.25.2(zod@4.4.3))(zod@4.4.3) - '@standard-community/standard-openapi': 0.2.9(@standard-community/standard-json@0.3.5(@standard-schema/spec@1.1.0)(@types/json-schema@7.0.15)(@valibot/to-json-schema@1.7.0(valibot@1.4.2))(quansync@0.2.11)(typebox@1.3.7)(valibot@1.4.2)(zod-to-json-schema@3.25.2(zod@4.4.3))(zod@4.4.3))(@standard-schema/spec@1.1.0)(openapi-types@12.1.3)(typebox@1.3.7)(valibot@1.4.2)(zod@4.4.3) - '@valibot/to-json-schema': 1.7.0(valibot@1.4.2) - hono: 4.12.31 - hono-openapi: 1.3.0(@hono/standard-validator@0.2.2(@standard-schema/spec@1.1.0)(hono@4.12.31))(@standard-community/standard-json@0.3.5(@standard-schema/spec@1.1.0)(@types/json-schema@7.0.15)(@valibot/to-json-schema@1.7.0(valibot@1.4.2))(quansync@0.2.11)(typebox@1.3.7)(valibot@1.4.2)(zod-to-json-schema@3.25.2(zod@4.4.3))(zod@4.4.3))(@standard-community/standard-openapi@0.2.9(@standard-community/standard-json@0.3.5(@standard-schema/spec@1.1.0)(@types/json-schema@7.0.15)(@valibot/to-json-schema@1.7.0(valibot@1.4.2))(quansync@0.2.11)(typebox@1.3.7)(valibot@1.4.2)(zod-to-json-schema@3.25.2(zod@4.4.3))(zod@4.4.3))(@standard-schema/spec@1.1.0)(openapi-types@12.1.3)(typebox@1.3.7)(valibot@1.4.2)(zod@4.4.3))(@types/json-schema@7.0.15)(hono@4.12.31)(openapi-types@12.1.3) - js-yaml: 4.1.1 - just-bash: 3.0.1 - openapi-types: 12.1.3 - quansync: 0.2.11 - ulidx: 2.4.1 - valibot: 1.4.2 - ws: 8.20.1 - transitivePeerDependencies: - - '@cfworker/json-schema' - - '@standard-schema/spec' - - '@types/json-schema' - - arktype - - bufferutil - - effect - - supports-color - - sury - - typebox - - typescript - - utf-8-validate - - zod - - zod-openapi - - zod-to-json-schema - - '@flue/runtime@1.0.0-beta.3(@standard-schema/spec@1.1.0)(@types/json-schema@7.0.15)(typebox@1.3.7)(ws@8.20.1)(zod-to-json-schema@3.25.2(zod@4.4.3))(zod@4.4.3)': + '@flue/runtime@1.0.0-beta.3(@standard-schema/spec@1.1.0)(@types/json-schema@7.0.15)(typebox@1.3.7)(ws@8.21.0)(zod-to-json-schema@3.25.2(zod@4.4.3))(zod@4.4.3)': dependencies: - '@earendil-works/pi-agent-core': 0.79.10(@modelcontextprotocol/sdk@1.29.0(zod@4.4.3))(ws@8.20.1)(zod@4.4.3) - '@earendil-works/pi-ai': 0.79.10(@modelcontextprotocol/sdk@1.29.0(zod@4.4.3))(ws@8.20.1)(zod@4.4.3) + '@earendil-works/pi-agent-core': 0.79.10(@modelcontextprotocol/sdk@1.29.0(zod@4.4.3))(ws@8.21.0)(zod@4.4.3) + '@earendil-works/pi-ai': 0.79.10(@modelcontextprotocol/sdk@1.29.0(zod@4.4.3))(ws@8.21.0)(zod@4.4.3) '@hono/node-server': 2.0.11(hono@4.12.31) '@hono/standard-validator': 0.2.2(@standard-schema/spec@1.1.0)(hono@4.12.31) '@modelcontextprotocol/sdk': 1.29.0(zod@4.4.3) @@ -3129,10 +2864,10 @@ snapshots: - zod-openapi - zod-to-json-schema - '@flue/runtime@1.0.0-beta.9(@standard-schema/spec@1.1.0)(@types/json-schema@7.0.15)(typebox@1.3.7)(ws@8.20.1)(zod-to-json-schema@3.25.2(zod@4.4.3))(zod@4.4.3)': + '@flue/runtime@1.0.0-beta.9(@standard-schema/spec@1.1.0)(@types/json-schema@7.0.15)(typebox@1.3.7)(ws@8.21.0)(zod-to-json-schema@3.25.2(zod@4.4.3))(zod@4.4.3)': dependencies: - '@earendil-works/pi-agent-core': 0.80.10(@modelcontextprotocol/sdk@1.29.0(zod@4.4.3))(ws@8.20.1)(zod@4.4.3) - '@earendil-works/pi-ai': 0.80.10(@modelcontextprotocol/sdk@1.29.0(zod@4.4.3))(ws@8.20.1)(zod@4.4.3) + '@earendil-works/pi-agent-core': 0.80.10(@modelcontextprotocol/sdk@1.29.0(zod@4.4.3))(ws@8.21.0)(zod@4.4.3) + '@earendil-works/pi-ai': 0.80.10(@modelcontextprotocol/sdk@1.29.0(zod@4.4.3))(ws@8.21.0)(zod@4.4.3) '@hono/node-server': 2.0.11(hono@4.12.31) '@hono/standard-validator': 0.2.2(@standard-schema/spec@1.1.0)(hono@4.12.31) '@modelcontextprotocol/sdk': 1.29.0(zod@4.4.3) @@ -3164,10 +2899,10 @@ snapshots: - zod-openapi - zod-to-json-schema - '@flue/runtime@2.0.0(@modelcontextprotocol/sdk@1.29.0(zod@4.4.3))(ws@8.20.1)(zod@4.4.3)': + '@flue/runtime@2.0.0(@modelcontextprotocol/sdk@1.29.0(zod@4.4.3))(ws@8.21.0)(zod@4.4.3)': dependencies: - '@earendil-works/pi-agent-core': 0.83.0(@modelcontextprotocol/sdk@1.29.0(zod@4.4.3))(ws@8.20.1)(zod@4.4.3) - '@earendil-works/pi-ai': 0.83.0(@modelcontextprotocol/sdk@1.29.0(zod@4.4.3))(ws@8.20.1)(zod@4.4.3) + '@earendil-works/pi-agent-core': 0.83.0(@modelcontextprotocol/sdk@1.29.0(zod@4.4.3))(ws@8.21.0)(zod@4.4.3) + '@earendil-works/pi-ai': 0.83.0(@modelcontextprotocol/sdk@1.29.0(zod@4.4.3))(ws@8.21.0)(zod@4.4.3) '@hono/node-server': 2.0.11(hono@4.12.31) '@modelcontextprotocol/client': 2.0.0 '@valibot/to-json-schema': 1.7.0(valibot@1.4.2) @@ -3184,10 +2919,10 @@ snapshots: - ws - zod - '@flue/runtime@2.0.1(@modelcontextprotocol/sdk@1.29.0(zod@4.4.3))(ws@8.20.1)(zod@4.4.3)': + '@flue/runtime@2.0.1(@modelcontextprotocol/sdk@1.29.0(zod@4.4.3))(ws@8.21.0)(zod@4.4.3)': dependencies: - '@earendil-works/pi-agent-core': 0.83.0(@modelcontextprotocol/sdk@1.29.0(zod@4.4.3))(ws@8.20.1)(zod@4.4.3) - '@earendil-works/pi-ai': 0.83.0(@modelcontextprotocol/sdk@1.29.0(zod@4.4.3))(ws@8.20.1)(zod@4.4.3) + '@earendil-works/pi-agent-core': 0.83.0(@modelcontextprotocol/sdk@1.29.0(zod@4.4.3))(ws@8.21.0)(zod@4.4.3) + '@earendil-works/pi-ai': 0.83.0(@modelcontextprotocol/sdk@1.29.0(zod@4.4.3))(ws@8.21.0)(zod@4.4.3) '@hono/node-server': 2.0.11(hono@4.12.31) '@modelcontextprotocol/client': 2.0.0 '@valibot/to-json-schema': 1.7.0(valibot@1.4.2) @@ -3407,7 +3142,7 @@ snapshots: '@img/sharp-wasm32@0.34.5': dependencies: - '@emnapi/runtime': 1.10.0 + '@emnapi/runtime': 1.11.3 optional: true '@img/sharp-wasm32@0.35.2': @@ -3467,15 +3202,6 @@ snapshots: '@microsoft/fetch-event-source@2.0.1': {} - '@mistralai/mistralai@2.2.1': - dependencies: - ws: 8.20.1 - zod: 4.4.3 - zod-to-json-schema: 3.25.2(zod@4.4.3) - transitivePeerDependencies: - - bufferutil - - utf-8-validate - '@mistralai/mistralai@2.2.6(@opentelemetry/api@1.9.0)': dependencies: '@opentelemetry/semantic-conventions': 1.43.0 @@ -4303,7 +4029,7 @@ snapshots: dependencies: '@cspotcode/source-map-support': 0.8.1 sharp: 0.35.2 - undici: 7.28.0 + undici: 7.29.0 workerd: 1.20260730.1 ws: 8.21.0 youch: 4.1.0-beta.10 @@ -4336,7 +4062,7 @@ snapshots: node-abi@3.92.0: dependencies: - semver: 7.8.0 + semver: 7.8.5 optional: true node-addon-api@8.7.0: @@ -4371,9 +4097,9 @@ snapshots: dependencies: wrappy: 1.0.2 - openai@6.26.0(ws@8.20.1)(zod@4.4.3): + openai@6.26.0(ws@8.21.0)(zod@4.4.3): optionalDependencies: - ws: 8.20.1 + ws: 8.21.0 zod: 4.4.3 openapi-types@12.1.3: {} @@ -4561,8 +4287,6 @@ snapshots: dependencies: commander: 6.2.1 - semver@7.8.0: {} - semver@7.8.5: {} send@1.2.1: @@ -4596,7 +4320,7 @@ snapshots: dependencies: '@img/colour': 1.1.0 detect-libc: 2.1.2 - semver: 7.8.0 + semver: 7.8.5 optionalDependencies: '@img/sharp-darwin-arm64': 0.34.5 '@img/sharp-darwin-x64': 0.34.5 @@ -4788,8 +4512,6 @@ snapshots: undici@7.29.0: {} - undici@7.28.0: {} - unenv@2.0.0-rc.24: dependencies: pathe: 2.0.3 diff --git a/e2e/scenarios/flue-instrumentation/scenario.cli.mjs b/e2e/scenarios/flue-instrumentation/scenario.cli.mjs deleted file mode 100644 index bbbb961fd..000000000 --- a/e2e/scenarios/flue-instrumentation/scenario.cli.mjs +++ /dev/null @@ -1,4 +0,0 @@ -const { runCliFlueInstrumentation, runMain } = - await import("./scenario.impl.mjs"); - -runMain(runCliFlueInstrumentation); diff --git a/e2e/scenarios/flue-instrumentation/scenario.impl.mjs b/e2e/scenarios/flue-instrumentation/scenario.impl.mjs index e0c4a5ce2..ba30b1543 100644 --- a/e2e/scenarios/flue-instrumentation/scenario.impl.mjs +++ b/e2e/scenarios/flue-instrumentation/scenario.impl.mjs @@ -1,5 +1,4 @@ import { spawn } from "node:child_process"; -import { access, rm } from "node:fs/promises"; import { createServer } from "node:net"; import path from "node:path"; import { fileURLToPath } from "node:url"; @@ -15,7 +14,6 @@ const flueCliPath = path.join( "bin", "flue.mjs", ); -const braintrustHookNodeOption = "--import=braintrust/hook.mjs"; function workflowPayload() { return { @@ -27,27 +25,23 @@ function workflowPayload() { }; } -export async function runNodeFlueInstrumentationScenario(options) { - const env = scenarioEnv(options); - const outputDir = path.join(process.cwd(), ".flue-build", options.outputName); - await runFlueCli( - [ - "build", - "--target", - "node", - "--root", - process.cwd(), - "--output", - outputDir, - ], - env, - ); +export async function runFlueInstrumentationScenario() { + const outputDir = path.join(process.cwd(), ".flue-build", "explicit"); + await runFlueCli([ + "build", + "--target", + "node", + "--root", + process.cwd(), + "--output", + outputDir, + ]); const port = await getFreePort(); const child = spawn(process.execPath, [path.join(outputDir, "server.mjs")], { cwd: process.cwd(), env: { - ...env, + ...process.env, PORT: String(port), }, stdio: ["ignore", "pipe", "pipe"], @@ -96,69 +90,10 @@ export async function runNodeFlueInstrumentationScenario(options) { } } -export async function runCliFlueInstrumentationScenario() { - const flushFile = path.join(process.cwd(), ".flue-build", "cli-flushed"); - await rm(flushFile, { force: true }); - await runFlueCli( - [ - "run", - "instrumentation", - "--target", - "node", - process.env.FLUE_E2E_INPUT_FLAG ?? "--payload", - JSON.stringify(workflowPayload()), - "--root", - process.cwd(), - ], - { - ...scenarioEnv({ autoHook: true, explicitObserve: false }), - FLUE_E2E_FLUSH_FILE: flushFile, - }, - ); - await waitForFile(flushFile); -} - -export function runExplicitFlueInstrumentation() { - return runNodeFlueInstrumentationScenario({ - autoHook: false, - explicitObserve: true, - outputName: "explicit", - }); -} - -export function runAutoFlueInstrumentation() { - return runNodeFlueInstrumentationScenario({ - autoHook: true, - explicitObserve: false, - outputName: "auto-hook", - }); -} - -export function runCliFlueInstrumentation() { - return runCliFlueInstrumentationScenario(); -} - -function scenarioEnv({ autoHook, explicitObserve }) { - const env = { - ...process.env, - FLUE_E2E_EXPLICIT_OBSERVE: explicitObserve ? "1" : "0", - }; - if (!autoHook) { - return env; - } - return { - ...env, - NODE_OPTIONS: [env.NODE_OPTIONS, braintrustHookNodeOption] - .filter(Boolean) - .join(" "), - }; -} - -async function runFlueCli(args, env) { +async function runFlueCli(args) { await new Promise((resolve, reject) => { const child = spawn(process.execPath, [flueCliPath, ...args], { cwd: process.cwd(), - env, stdio: ["ignore", "pipe", "pipe"], }); @@ -245,27 +180,3 @@ async function stopChild(child) { }, 5_000).unref(); }); } - -async function waitForFile(filePath) { - const startedAt = Date.now(); - while (true) { - try { - await access(filePath); - return; - } catch { - if (Date.now() - startedAt > 30_000) { - throw new Error( - `timed out waiting for Flue e2e flush marker: ${filePath}`, - ); - } - await new Promise((resolve) => setTimeout(resolve, 50)); - } - } -} - -export function runMain(main) { - void main().catch((error) => { - console.error(error); - process.exitCode = 1; - }); -} diff --git a/e2e/scenarios/flue-instrumentation/scenario.mjs b/e2e/scenarios/flue-instrumentation/scenario.mjs deleted file mode 100644 index a9bd9d94e..000000000 --- a/e2e/scenarios/flue-instrumentation/scenario.mjs +++ /dev/null @@ -1,8 +0,0 @@ -import "braintrust"; - -await new Promise((resolve) => setImmediate(resolve)); - -const { runAutoFlueInstrumentation, runMain } = - await import("./scenario.impl.mjs"); - -runMain(runAutoFlueInstrumentation); diff --git a/e2e/scenarios/flue-instrumentation/scenario.test.ts b/e2e/scenarios/flue-instrumentation/scenario.test.ts index 73a27bbf7..22d9941be 100644 --- a/e2e/scenarios/flue-instrumentation/scenario.test.ts +++ b/e2e/scenarios/flue-instrumentation/scenario.test.ts @@ -10,49 +10,19 @@ import { defineFlueInstrumentationAssertions } from "./assertions"; import { defineFlueV2InstrumentationAssertions } from "./v2-assertions"; const originalScenarioDir = resolveScenarioDir(import.meta.url); -const generatedScenarioRoot = path.resolve( - originalScenarioDir, - "../../.bt-tmp/generated-scenarios/flue-instrumentation", -); const TIMEOUT_MS = 120_000; -const flueV1ScenarioSourceDir = await prepareGeneratedFlueV1ScenarioSource(); const flueScenarios = await Promise.all([ prepareFlueScenario({ cliPackageName: "@flue/cli", - expectAmbientContext: false, - label: "v0.8 pinned", - runtimePackageName: "@flue/runtime", - sourceDir: originalScenarioDir, - supportsAutoInstrumentation: true, - variantKey: "flue-v0-8-0", - }), - prepareFlueScenario({ - cliPackageName: "flue-cli-v0-8-latest", - expectAmbientContext: false, - label: "v0.8 latest", - modelName: "openai/gpt-5.4-nano", - runtimePackageName: "flue-runtime-v0-8-latest", - sourceDir: originalScenarioDir, - supportsAutoInstrumentation: true, - variantKey: "flue-v0-8-latest", - }), - prepareFlueScenario({ - cliPackageName: "flue-cli-v1", - expectAmbientContext: true, label: "v1 pinned", - runtimePackageName: "flue-runtime-v1", - sourceDir: flueV1ScenarioSourceDir, - supportsAutoInstrumentation: false, + runtimePackageName: "@flue/runtime", variantKey: "flue-v1-0-0-beta-3", }), prepareFlueScenario({ cliPackageName: "flue-cli-v1-latest", - expectAmbientContext: true, label: "v1 latest", modelName: "openai/gpt-5.4-nano", runtimePackageName: "flue-runtime-v1-latest", - sourceDir: flueV1ScenarioSourceDir, - supportsAutoInstrumentation: false, variantKey: "flue-v1-latest", }), ]); @@ -84,7 +54,6 @@ describe.sequential("flue variants", () => { for (const scenario of flueScenarios) { describe.sequential(`flue ${scenario.label} (${scenario.version})`, () => { defineFlueInstrumentationAssertions({ - expectAmbientContext: scenario.expectAmbientContext, name: "explicit instrumentation", runScenario: async ({ runScenarioDir }) => { await runScenarioDir({ @@ -102,54 +71,6 @@ describe.sequential("flue variants", () => { testFileUrl: import.meta.url, timeoutMs: TIMEOUT_MS, }); - - if (scenario.supportsAutoInstrumentation) { - defineFlueInstrumentationAssertions({ - expectAmbientContext: scenario.expectAmbientContext, - name: "auto-hook instrumentation", - runScenario: async ({ runNodeScenarioDir }) => { - await runNodeScenarioDir({ - entry: "scenario.mjs", - env: scenario.env, - nodeArgs: ["--import", "braintrust/hook.mjs"], - runContext: { - originalScenarioDir, - variantKey: scenario.variantKey, - }, - scenarioDir: scenario.scenarioDir, - timeoutMs: TIMEOUT_MS, - }); - }, - snapshotName: `${scenario.variantKey}-auto-hook`, - testFileUrl: import.meta.url, - timeoutMs: TIMEOUT_MS, - }); - - defineFlueInstrumentationAssertions({ - expectAmbientContext: scenario.expectAmbientContext, - name: "cli instrumentation", - runScenario: async ({ runNodeScenarioDir }) => { - await runNodeScenarioDir({ - entry: "scenario.cli.mjs", - env: { - ...scenario.env, - ...(scenario.inputFlag - ? { FLUE_E2E_INPUT_FLAG: scenario.inputFlag } - : {}), - }, - runContext: { - originalScenarioDir, - variantKey: scenario.variantKey, - }, - scenarioDir: scenario.scenarioDir, - timeoutMs: TIMEOUT_MS, - }); - }, - snapshotName: `${scenario.variantKey}-cli`, - testFileUrl: import.meta.url, - timeoutMs: TIMEOUT_MS, - }); - } }); } @@ -169,17 +90,13 @@ describe.sequential("flue variants", () => { async function prepareFlueScenario(options: { cliPackageName: string; - expectAmbientContext: boolean; - inputFlag?: string; label: string; modelName?: string; runtimePackageName: string; - sourceDir: string; - supportsAutoInstrumentation: boolean; variantKey: string; }) { const scenarioDir = await prepareScenarioDir({ - scenarioDir: options.sourceDir, + scenarioDir: originalScenarioDir, }); const [cliPackageDir, runtimePackageDir] = await Promise.all([ fs.realpath(path.join(scenarioDir, "node_modules", options.cliPackageName)), @@ -203,7 +120,6 @@ async function prepareFlueScenario(options: { ), ]); return { - ...options, env: { FLUE_CLI_PACKAGE_NAME: options.cliPackageName, FLUE_RUNTIME_PACKAGE_NAME: options.runtimePackageName, @@ -216,39 +132,12 @@ async function prepareFlueScenario(options: { } : {}), }, + label: options.label, scenarioDir, + variantKey: options.variantKey, version: await readInstalledPackageVersion( scenarioDir, options.runtimePackageName, ), }; } - -async function prepareGeneratedFlueV1ScenarioSource() { - const sourceDir = path.join(generatedScenarioRoot, "flue-v1"); - await fs.rm(sourceDir, { force: true, recursive: true }); - await fs.mkdir(sourceDir, { recursive: true }); - await fs.cp(originalScenarioDir, sourceDir, { - filter(source) { - const relative = path.relative(originalScenarioDir, source); - return ( - relative === "" || - !["__cassettes__", "__snapshots__", "node_modules", "versions"].some( - (name) => - relative === name || relative.startsWith(`${name}${path.sep}`), - ) - ); - }, - recursive: true, - }); - await fs.cp(path.join(originalScenarioDir, "versions", "v1"), sourceDir, { - filter(source) { - return ( - source !== - path.join(originalScenarioDir, "versions", "v1", "package.json") - ); - }, - recursive: true, - }); - return sourceDir; -} diff --git a/e2e/scenarios/flue-instrumentation/scenario.ts b/e2e/scenarios/flue-instrumentation/scenario.ts index b96ed4882..a5e57e15d 100644 --- a/e2e/scenarios/flue-instrumentation/scenario.ts +++ b/e2e/scenarios/flue-instrumentation/scenario.ts @@ -1,3 +1,3 @@ -import { runExplicitFlueInstrumentation, runMain } from "./scenario.impl.mjs"; +import { runFlueInstrumentationScenario } from "./scenario.impl.mjs"; -runMain(runExplicitFlueInstrumentation); +await runFlueInstrumentationScenario(); diff --git a/e2e/scenarios/flue-instrumentation/versions/v1/.flue/app.mjs b/e2e/scenarios/flue-instrumentation/versions/v1/.flue/app.mjs deleted file mode 100644 index 6680c7ae8..000000000 --- a/e2e/scenarios/flue-instrumentation/versions/v1/.flue/app.mjs +++ /dev/null @@ -1,112 +0,0 @@ -import { flush, initLogger } from "braintrust"; -import { Hono } from "hono"; -import { mkdir, writeFile } from "node:fs/promises"; -import { dirname } from "node:path"; - -const runtimePackageName = - process.env.FLUE_RUNTIME_PACKAGE_NAME ?? "flue-runtime-v1"; -const [{ instrument, registerProvider }, { flue }] = await Promise.all([ - import(runtimePackageName), - import(`${runtimePackageName}/routing`), -]); - -function projectName() { - const configured = process.env.BRAINTRUST_E2E_PROJECT_NAME; - if (configured) { - return configured; - } - const testRunId = process.env.BRAINTRUST_E2E_RUN_ID ?? "local"; - return `e2e-flue-instrumentation-${testRunId.toLowerCase().replace(/[^a-z0-9-]/g, "-")}`; -} - -initLogger({ projectName: projectName() }); - -const exitProcess = process.exit.bind(process); -if (process.env.FLUE_E2E_FLUSH_FILE) { - let isExiting = false; - process.exit = (code) => { - if (isExiting) { - return exitProcess(code); - } - isExiting = true; - const keepAlive = setTimeout(() => {}, 30_000); - void flushBeforeExit() - .catch((error) => { - console.error(error); - }) - .finally(() => { - clearTimeout(keepAlive); - exitProcess(code); - }); - }; -} - -if (process.env.FLUE_E2E_EXPLICIT_OBSERVE === "1") { - const { braintrustFlueInstrumentation } = await import("braintrust"); - instrument(braintrustFlueInstrumentation()); -} - -const openAIBaseUrl = - process.env.OPENAI_BASE_URL ?? process.env.BRAINTRUST_E2E_MODEL_BASE_URL; -if (openAIBaseUrl) { - registerProvider("openai", { baseUrl: openAIBaseUrl }); -} - -const anthropicBaseUrl = process.env.ANTHROPIC_BASE_URL; -if (anthropicBaseUrl) { - registerProvider("anthropic", { - apiKey: process.env.ANTHROPIC_API_KEY ?? "test-key", - baseUrl: anthropicBaseUrl, - }); -} - -let didScheduleFlush = false; -function scheduleFinalFlush(exitAfterFlush = false) { - if (didScheduleFlush) { - return; - } - didScheduleFlush = true; - const keepAlive = setTimeout(() => {}, 30_000); - void flushBeforeExit() - .catch((error) => { - console.error(error); - }) - .finally(() => { - clearTimeout(keepAlive); - if (exitAfterFlush) { - exitProcess(0); - } - }); -} - -process.on("SIGTERM", () => { - scheduleFinalFlush(true); -}); - -process.on("beforeExit", () => { - scheduleFinalFlush(); -}); - -const app = new Hono(); -app.route("/", flue()); - -async function flushBeforeExit() { - await new Promise((resolve) => setTimeout(resolve, 250)); - await flush(); - if (process.env.FLUE_E2E_FLUSH_FILE) { - await mkdir(dirname(process.env.FLUE_E2E_FLUSH_FILE), { - recursive: true, - }); - await writeFile(process.env.FLUE_E2E_FLUSH_FILE, "ok"); - } -} - -export default { - async fetch(request, env, ctx) { - if (new URL(request.url).pathname === "/__braintrust_flush") { - await flush(); - return new Response("ok"); - } - return app.fetch(request, env, ctx); - }, -}; diff --git a/e2e/scenarios/flue-instrumentation/versions/v1/.flue/workflows/instrumentation.mjs b/e2e/scenarios/flue-instrumentation/versions/v1/.flue/workflows/instrumentation.mjs deleted file mode 100644 index 2c8660430..000000000 --- a/e2e/scenarios/flue-instrumentation/versions/v1/.flue/workflows/instrumentation.mjs +++ /dev/null @@ -1,183 +0,0 @@ -import { traced } from "braintrust"; -import * as v from "valibot"; -import { - FLUE_MODEL, - FLUE_REASONING_MODEL, - SCENARIO_NAME, -} from "../../constants.mjs"; - -const runtimePackageName = - process.env.FLUE_RUNTIME_PACKAGE_NAME ?? "flue-runtime-v1"; -const [{ defineAgent, defineTool, defineWorkflow }, { local }] = - await Promise.all([ - import(runtimePackageName), - import(`${runtimePackageName}/node`), - ]); - -function flueModel() { - return process.env.FLUE_E2E_MODEL ?? FLUE_MODEL; -} - -function flueReasoningModel() { - return process.env.FLUE_E2E_REASONING_MODEL ?? FLUE_REASONING_MODEL; -} - -function fluePromptModel() { - return process.env.FLUE_E2E_PROMPT_MODEL ?? flueReasoningModel(); -} - -function fluePromptThinkingLevel() { - return ( - process.env.FLUE_E2E_PROMPT_THINKING_LEVEL ?? flueReasoningThinkingLevel() - ); -} - -function flueReasoningThinkingLevel() { - return process.env.FLUE_E2E_REASONING_THINKING_LEVEL ?? "medium"; -} - -const flueE2EAgent = defineAgent(() => ({ - compaction: { - keepRecentTokens: 1, - reserveTokens: 64, - }, - cwd: process.cwd(), - instructions: [ - "You are a deterministic Flue instrumentation test agent.", - "Follow user instructions exactly.", - "When asked for a marker, output only that marker and no extra text.", - "When running a local skill file, read it yourself and do not delegate it to a task.", - ].join(" "), - model: flueModel(), - sandbox: local({ cwd: process.cwd() }), - thinkingLevel: "off", -})); - -const lookupTool = defineTool({ - description: - "Return a deterministic lookup result with an id needed by web_search.", - input: v.object({ - query: v.string(), - }), - name: "lookup", - run: async ({ input }) => { - await traced( - async (span) => { - span.log({ output: "lookup-active" }); - }, - { - name: "flue.toolCurrentProbe", - event: { - metadata: { - scenario: SCENARIO_NAME, - }, - }, - }, - ); - - return { - id: "flue-session-2026", - query: input.query, - topic: "session instrumentation", - }; - }, -}); - -const webSearchTool = defineTool({ - description: - "Search a deterministic local web index. Requires the id returned by lookup.", - input: v.object({ - lookupId: v.string(), - query: v.string(), - }), - name: "web_search", - run: async ({ input }) => ({ - lookupId: input.lookupId, - query: input.query, - results: [ - { - title: "Flue reasoning stream instrumentation", - url: "https://example.test/flue/reasoning-streams", - }, - ], - }), -}); - -const summarizeSourceTool = defineTool({ - description: - "Summarize the selected deterministic source after web_search returns a URL.", - input: v.object({ - url: v.string(), - }), - name: "summarize_source", - run: async ({ input }) => ({ - summary: - "Flue emits reasoning, tool execution, and LLM turn events separately.", - url: input.url, - }), -}); - -export async function route(_ctx, next) { - await next(); -} - -export default defineWorkflow({ - agent: flueE2EAgent, - input: v.object({ - scenario: v.optional(v.string()), - metadata: v.optional(v.record(v.string(), v.unknown())), - }), - async run({ harness, input }) { - await traced( - async (span) => { - span.log({ output: "active" }); - }, - { - name: "flue.workflowCurrentProbe", - event: { - metadata: { - scenario: SCENARIO_NAME, - }, - }, - }, - ); - - const session = await harness.session("main"); - const skillSession = await harness.session("skill"); - const taskSession = await harness.session("task"); - - await session.prompt( - [ - "Complete this instrumented research flow.", - "Call exactly one tool per turn and wait for each tool result before choosing the next tool.", - 'Step 1: call lookup with query "flue instrumentation".', - 'Step 2: use the lookup result id as lookupId and call web_search with query "Braintrust Flue reasoning stream instrumentation".', - "Step 3: use the first web_search result url and call summarize_source.", - "After summarize_source returns, reply with exactly PROMPT_DONE and no other text.", - ].join(" "), - { - model: fluePromptModel(), - thinkingLevel: fluePromptThinkingLevel(), - tools: [lookupTool, webSearchTool, summarizeSourceTool], - }, - ); - - await skillSession.skill("e2e-flue-skill", { - args: { marker: "SKILL_DONE" }, - model: flueReasoningModel(), - thinkingLevel: "off", - }); - - await taskSession.task("Reply with exactly TASK_DONE and no other text.", { - model: FLUE_MODEL, - thinkingLevel: "off", - }); - - await session.compact(); - - return { - scenario: input.scenario ?? SCENARIO_NAME, - status: "done", - }; - }, -}); diff --git a/js/src/auto-instrumentations/configs/all.ts b/js/src/auto-instrumentations/configs/all.ts index 23d58969a..3b42cb11a 100644 --- a/js/src/auto-instrumentations/configs/all.ts +++ b/js/src/auto-instrumentations/configs/all.ts @@ -13,7 +13,6 @@ import { cloudflareAgentsConfigs } from "./cloudflare-agents"; import { cloudflareThinkConfigs } from "./cloudflare-think"; import { cohereConfigs } from "./cohere"; import { cursorSDKConfigs } from "./cursor-sdk"; -import { flueConfigs } from "./flue"; import { genkitConfigs } from "./genkit"; import { gitHubCopilotConfigs } from "./github-copilot"; import { googleADKConfigs } from "./google-adk"; @@ -115,10 +114,6 @@ const defaultInstrumentationConfigGroups: readonly InstrumentationConfigGroup[] integrations: ["strandsAgentSDK"], configs: strandsAgentSDKConfigs, }, - { - integrations: ["flue"], - configs: flueConfigs, - }, { integrations: ["voyageai"], configs: voyageAIConfigs, diff --git a/js/src/auto-instrumentations/configs/flue.test.ts b/js/src/auto-instrumentations/configs/flue.test.ts deleted file mode 100644 index 456b5110f..000000000 --- a/js/src/auto-instrumentations/configs/flue.test.ts +++ /dev/null @@ -1,63 +0,0 @@ -import { create } from "@apm-js-collab/code-transformer"; -import { describe, expect, it } from "vitest"; -import { flueConfigs, flueVersionRange } from "./flue"; - -describe("flue auto-instrumentation configs", () => { - it("targets the stable Flue 0.8 context factory", () => { - const matcher = create(flueConfigs); - const flue08Transformer = matcher.getTransformer( - "@flue/runtime", - "0.8.0", - "dist/internal.mjs", - ); - const flue10BetaTransformer = matcher.getTransformer( - "@flue/runtime", - "1.0.0-beta.3", - "dist/internal.mjs", - ); - const flue10Transformer = matcher.getTransformer( - "@flue/runtime", - "1.0.0", - "dist/internal.mjs", - ); - - expect(flueVersionRange).toBe(">=0.8.0 <1.0.0"); - expect(flue08Transformer).toBeDefined(); - expect(flue10BetaTransformer).toBeUndefined(); - expect(flue10Transformer).toBeUndefined(); - const transformed = flue08Transformer!.transform( - ` -function createFlueContext(config) { - return { config }; -} -`, - "esm", - ).code; - - expect(transformed).toContain( - "orchestrion:@flue/runtime:createFlueContext", - ); - }); - - it("does not target Flue content-hashed workflow or tool chunks", () => { - const matcher = create(flueConfigs); - - expect( - matcher.getTransformer( - "@flue/runtime", - "0.8.0", - "dist/handle-agent-DcUclCE2.mjs", - ), - ).toBeUndefined(); - expect( - matcher.getTransformer( - "@flue/runtime", - "0.8.0", - "dist/sandbox-DNEJXjr_.mjs", - ), - ).toBeUndefined(); - expect( - matcher.getTransformer("@flue/runtime", "1.0.0-beta.3", "dist/index.mjs"), - ).toBeUndefined(); - }); -}); diff --git a/js/src/auto-instrumentations/configs/flue.ts b/js/src/auto-instrumentations/configs/flue.ts deleted file mode 100644 index a04c02bf2..000000000 --- a/js/src/auto-instrumentations/configs/flue.ts +++ /dev/null @@ -1,19 +0,0 @@ -import type { InstrumentationConfig } from "../orchestrion-js"; -import { flueChannels } from "../../instrumentation/plugins/flue-channels"; - -export const flueVersionRange = ">=0.8.0 <1.0.0"; - -export const flueConfigs: InstrumentationConfig[] = [ - { - channelName: flueChannels.createContext.channelName, - module: { - name: "@flue/runtime", - versionRange: flueVersionRange, - filePath: "dist/internal.mjs", - }, - functionQuery: { - functionName: "createFlueContext", - kind: "Sync", - }, - }, -]; diff --git a/js/src/exports.ts b/js/src/exports.ts index 2f4332842..e8bac9cc2 100644 --- a/js/src/exports.ts +++ b/js/src/exports.ts @@ -351,8 +351,5 @@ export { // Auto-instrumentation configuration export { configureInstrumentation } from "./instrumentation"; -export { - braintrustFlueObserver, - braintrustFlueInstrumentation, -} from "./instrumentation"; +export { braintrustFlueInstrumentation } from "./instrumentation"; export type { InstrumentationConfig } from "./instrumentation"; diff --git a/js/src/instrumentation/braintrust-plugin.ts b/js/src/instrumentation/braintrust-plugin.ts index 4ed98d53e..53bebc00d 100644 --- a/js/src/instrumentation/braintrust-plugin.ts +++ b/js/src/instrumentation/braintrust-plugin.ts @@ -20,7 +20,6 @@ import { GroqPlugin } from "./plugins/groq-plugin"; import { BedrockRuntimePlugin } from "./plugins/bedrock-runtime-plugin"; import { GenkitPlugin } from "./plugins/genkit-plugin"; import { GitHubCopilotPlugin } from "./plugins/github-copilot-plugin"; -import { FluePlugin } from "./plugins/flue-plugin"; import { LangChainPlugin } from "./plugins/langchain-plugin"; import { LangSmithPlugin } from "./plugins/langsmith-plugin"; import { PiCodingAgentPlugin } from "./plugins/pi-coding-agent-plugin"; @@ -77,7 +76,6 @@ export class BraintrustPlugin extends BasePlugin { private bedrockRuntimePlugin: BedrockRuntimePlugin | null = null; private genkitPlugin: GenkitPlugin | null = null; private gitHubCopilotPlugin: GitHubCopilotPlugin | null = null; - private fluePlugin: FluePlugin | null = null; private langChainPlugin: LangChainPlugin | null = null; private langSmithPlugin: LangSmithPlugin | null = null; private piCodingAgentPlugin: PiCodingAgentPlugin | null = null; @@ -234,11 +232,6 @@ export class BraintrustPlugin extends BasePlugin { this.cloudflareAgentsPlugin.enable(); } - if (integrations.flue !== false) { - this.fluePlugin = new FluePlugin(); - this.fluePlugin.enable(); - } - if (integrations.langchain !== false && integrations.langgraph !== false) { this.langChainPlugin = new LangChainPlugin(); this.langChainPlugin.enable(); @@ -389,11 +382,6 @@ export class BraintrustPlugin extends BasePlugin { this.cloudflareAgentsPlugin = null; } - if (this.fluePlugin) { - this.fluePlugin.disable(); - this.fluePlugin = null; - } - if (this.langChainPlugin) { this.langChainPlugin.disable(); this.langChainPlugin = null; diff --git a/js/src/instrumentation/config.ts b/js/src/instrumentation/config.ts index 35d3d2cb3..e8df49037 100644 --- a/js/src/instrumentation/config.ts +++ b/js/src/instrumentation/config.ts @@ -12,7 +12,6 @@ export interface InstrumentationIntegrationsConfig { cloudflareThink?: boolean; cursor?: boolean; cursorSDK?: boolean; - flue?: boolean; mastra?: boolean; openAIAgents?: boolean; openrouter?: boolean; @@ -80,8 +79,6 @@ const envIntegrationAliases: Record< cursor: "cursor", "cursor-sdk": "cursorSDK", cursorsdk: "cursorSDK", - flue: "flue", - "flue-runtime": "flue", mastra: "mastra", "openai-agents": "openAIAgents", openaiagents: "openAIAgents", @@ -143,7 +140,6 @@ export function getDefaultInstrumentationIntegrations(): Record< cloudflareThink: true, cursor: true, cursorSDK: true, - flue: true, mastra: true, openAIAgents: true, openrouter: true, diff --git a/js/src/instrumentation/index.ts b/js/src/instrumentation/index.ts index 338d62abd..ce4352ebc 100644 --- a/js/src/instrumentation/index.ts +++ b/js/src/instrumentation/index.ts @@ -19,10 +19,7 @@ export { BraintrustPlugin } from "./braintrust-plugin"; export type { BraintrustPluginConfig } from "./braintrust-plugin"; export { OpenAIAgentsTraceProcessor } from "./plugins/openai-agents-trace-processor"; export type { OpenAIAgentsTraceProcessorOptions } from "./plugins/openai-agents-trace-processor"; -export { - braintrustFlueInstrumentation, - braintrustFlueObserver, -} from "./plugins/flue-plugin"; +export { braintrustFlueInstrumentation } from "./plugins/flue-plugin"; export { braintrustEveHook, braintrustEveInstrumentation, diff --git a/js/src/instrumentation/plugins/flue-channels.ts b/js/src/instrumentation/plugins/flue-channels.ts deleted file mode 100644 index 308d50aea..000000000 --- a/js/src/instrumentation/plugins/flue-channels.ts +++ /dev/null @@ -1,14 +0,0 @@ -import { channel, defineChannels } from "../core/channel-definitions"; -import { INSTRUMENTATION_NAMES } from "../../span-origin"; -import type { FlueObservableContext } from "../../vendor-sdk-types/flue"; - -export const flueChannels = defineChannels( - "@flue/runtime", - { - createContext: channel<[unknown], FlueObservableContext>({ - channelName: "createFlueContext", - kind: "sync-stream", - }), - }, - { instrumentationName: INSTRUMENTATION_NAMES.FLUE }, -); diff --git a/js/src/instrumentation/plugins/flue-plugin.test.ts b/js/src/instrumentation/plugins/flue-plugin.test.ts index 071b2a1b4..04901f8cc 100644 --- a/js/src/instrumentation/plugins/flue-plugin.test.ts +++ b/js/src/instrumentation/plugins/flue-plugin.test.ts @@ -37,43 +37,6 @@ vi.mock("../../debug-logger", () => ({ }, })); -const { mockNewTracingChannel, mockTracingChannels } = vi.hoisted(() => { - const tracingChannels = new Map(); - - function tracingChannel(name: string) { - const existing = tracingChannels.get(name); - if (existing) { - return existing; - } - - const handlers = new Set(); - const stores = new Map unknown>(); - const channel = { - __handlers: handlers, - __stores: stores, - start: { - bindStore: vi.fn( - (store: unknown, transform: (message: any) => unknown) => { - stores.set(store, transform); - }, - ), - unbindStore: vi.fn((store: unknown) => stores.delete(store)), - }, - subscribe: vi.fn((handler: any) => { - handlers.add(handler); - }), - unsubscribe: vi.fn((handler: any) => handlers.delete(handler)), - }; - tracingChannels.set(name, channel); - return channel; - } - - return { - mockNewTracingChannel: vi.fn((name: string) => tracingChannel(name)), - mockTracingChannels: tracingChannels, - }; -}); - vi.mock("../../logger", () => ({ BRAINTRUST_CURRENT_SPAN_STORE: mockCurrentSpanStoreSymbol, NOOP_SPAN: {}, @@ -101,24 +64,9 @@ vi.mock("../../logger", () => ({ }, })); -vi.mock("../../isomorph", () => ({ - default: { - newTracingChannel: mockNewTracingChannel, - }, -})); - -import { - FluePlugin, - braintrustFlueInstrumentation, - braintrustFlueObserver, -} from "./flue-plugin"; - -type Subscriber = typeof braintrustFlueObserver; - -const CREATE_CONTEXT_CHANNEL_NAME = - "orchestrion:@flue/runtime:createFlueContext"; +import { braintrustFlueInstrumentation } from "./flue-plugin"; -describe("Flue observe instrumentation", () => { +describe("Flue instrumentation", () => { let spans: Array<{ args: any; end: ReturnType; @@ -160,52 +108,20 @@ describe("Flue observe instrumentation", () => { }); afterEach(() => { - delete (globalThis as Record)[ - Symbol.for("braintrust.flue.auto-state") - ]; delete (globalThis as Record)[ Symbol.for("braintrust.flue.observe-bridge") ]; - for (const channel of mockTracingChannels.values()) { - channel.__handlers.clear(); - channel.__stores.clear(); - } vi.clearAllMocks(); }); - it("exports a subscriber that can be passed directly to Flue observe", () => { - const subscribers: Subscriber[] = []; - const unsubscribe = vi.fn(); - const observe = vi.fn((subscriber: Subscriber) => { - subscribers.push(subscriber); - return unsubscribe; - }); - - const unregister = observe(braintrustFlueObserver); - - expect(observe).toHaveBeenCalledTimes(1); - expect(observe).toHaveBeenCalledWith(braintrustFlueObserver); - expect(subscribers).toHaveLength(1); - - subscribers[0]?.({ - runId: "run-1", - type: "run_start", - workflowName: "research", - }); - expect(findSpan("workflow:research")).toBeDefined(); - - unregister(); - expect(unsubscribe).toHaveBeenCalledTimes(1); - }); - - it("exports a Flue 1.0 instrumentation factory", async () => { + it("exports a Flue instrumentation factory", async () => { const instrumentation = braintrustFlueInstrumentation(); const instrument = vi.fn((value: typeof instrumentation) => value); const registered = instrument(instrumentation); expect(instrument).toHaveBeenCalledWith(instrumentation); - expect(registered.observe).toBe(braintrustFlueObserver); + expect(typeof registered.observe).toBe("function"); expect(registered.key).toBe(Symbol.for("braintrust.flue.instrumentation")); expect(typeof registered.interceptor).toBe("function"); expect(() => registered.dispose()).not.toThrow(); @@ -218,7 +134,7 @@ describe("Flue observe instrumentation", () => { type: "workflow", workflowName: "research", }, - { eventContext: { id: "ctx-1", runId: "run-1" } }, + { eventContext: { id: "ctx-1" } }, async () => mockStartSpan({ name: "app.phase" }), ); const workflowSpan = findSpan("workflow:research"); @@ -232,37 +148,7 @@ describe("Flue observe instrumentation", () => { expect("enterWith" in mockCurrentSpanStore).toBe(false); }); - it("keeps the legacy observer compatible with Flue 1.0 instrumentation", async () => { - expect(braintrustFlueObserver.observe).toBe(braintrustFlueObserver); - expect(braintrustFlueObserver.key).toBe( - Symbol.for("braintrust.flue.instrumentation"), - ); - expect(typeof braintrustFlueObserver.interceptor).toBe("function"); - expect(() => braintrustFlueObserver.dispose()).not.toThrow(); - - const appSpan = await braintrustFlueObserver.interceptor( - { - phase: "start", - runId: "run-1", - startedAt: "2026-05-27T05:12:31.000Z", - type: "workflow", - workflowName: "research", - }, - { eventContext: { id: "ctx-1", runId: "run-1" } }, - async () => mockStartSpan({ name: "app.phase" }), - ); - const workflowSpan = findSpan("workflow:research"); - - expect(workflowSpan).toBeDefined(); - expect(appSpan.spanParents).toEqual([workflowSpan?.spanId]); - expect(mockCurrentSpanStore.run).toHaveBeenCalledWith( - workflowSpan, - expect.any(Function), - ); - expect("enterWith" in mockCurrentSpanStore).toBe(false); - }); - - it("maps Flue 0.8 observe events into semantic Braintrust spans", () => { + it("maps Flue 1 workflow observations into semantic Braintrust spans", () => { const emit = observeEvents(); const usage = flueUsage(); const startedAt = "2026-05-27T05:12:31.000Z"; @@ -270,8 +156,7 @@ describe("Flue observe instrumentation", () => { emit( { instanceId: "instance-1", - owner: { kind: "workflow", workflowName: "research" }, - payload: { + input: { metadata: { scenario: "flue-instrumentation", testRunId: "e2e-run-1", @@ -284,7 +169,7 @@ describe("Flue observe instrumentation", () => { type: "run_start", workflowName: "research", }, - { id: "ctx-1", runId: "run-1" }, + { id: "ctx-1" }, ); emit({ operationId: "op-1", @@ -295,17 +180,19 @@ describe("Flue observe instrumentation", () => { type: "operation_start", }); emit({ - api: "responses", - input: { - messages: [{ content: "Find Flue changes", role: "user" }], - systemPrompt: "Be precise", - tools: [{ name: "lookup", parameters: {} }], - }, - model: "claude-test", operationId: "op-1", - provider: "anthropic", purpose: "agent", - reasoning: "medium", + request: { + api: "responses", + input: { + messages: [{ content: "Find Flue changes", role: "user" }], + systemPrompt: "Be precise", + tools: [{ name: "lookup", parameters: {} }], + }, + providerName: "anthropic", + reasoningLevel: "medium", + requestedModel: "claude-test", + }, runId: "run-1", timestamp: "2026-05-27T05:12:33.000Z", turnId: "turn-1", @@ -331,23 +218,30 @@ describe("Flue observe instrumentation", () => { toolCallId: "tool-1", toolName: "lookup", turnId: "turn-1", - type: "tool_call", + type: "tool", }); emit({ - api: "responses", durationMs: 12, isError: false, - model: "claude-test", operationId: "op-1", - output: { content: [{ text: "done", type: "text" }], role: "assistant" }, - provider: "anthropic", purpose: "agent", + request: { + api: "responses", + providerName: "anthropic", + requestedModel: "claude-test", + }, + response: { + output: { + content: [{ text: "done", type: "text" }], + role: "assistant", + }, + finishReason: "stop", + usage, + }, runId: "run-1", - stopReason: "stop", timestamp: "2026-05-27T05:12:36.000Z", turnId: "turn-1", type: "turn", - usage, }); emit({ agent: "worker", @@ -493,150 +387,6 @@ describe("Flue observe instrumentation", () => { expect(mockFlush).toHaveBeenCalledTimes(1); }); - it("maps Flue 1.0 observations into semantic Braintrust spans", () => { - const emit = observeEvents(); - const usage = flueUsage(); - - emit( - { - eventIndex: 0, - input: { - metadata: { scenario: "flue-v1" }, - topic: "native instrumentation", - }, - runId: "run-1", - timestamp: "2026-05-27T05:12:31.000Z", - type: "run_start", - v: 3, - workflowName: "research", - }, - { id: "ctx-1", runId: "run-1" }, - ); - emit({ - eventIndex: 1, - operationId: "op-1", - operationKind: "prompt", - runId: "run-1", - type: "operation_start", - v: 3, - }); - emit({ - eventIndex: 2, - operationId: "op-1", - purpose: "agent", - request: { - api: "responses", - input: { - messages: [{ content: "Find native hooks", role: "user" }], - systemPrompt: "Be exact", - tools: [{ name: "lookup" }], - }, - model: "claude-test", - providerId: "anthropic", - providerName: "anthropic", - reasoning: "medium", - }, - runId: "run-1", - turnId: "turn-1", - type: "turn_request", - v: 3, - }); - emit({ - args: { query: "native flue instrumentation" }, - eventIndex: 3, - operationId: "op-1", - runId: "run-1", - toolCallId: "tool-1", - toolName: "lookup", - turnId: "turn-1", - type: "tool_start", - v: 3, - }); - emit({ - durationMs: 4, - eventIndex: 4, - isError: false, - operationId: "op-1", - output: { ok: true }, - runId: "run-1", - toolCallId: "tool-1", - toolName: "lookup", - turnId: "turn-1", - type: "tool", - v: 3, - }); - emit({ - durationMs: 12, - eventIndex: 5, - isError: false, - operationId: "op-1", - purpose: "agent", - request: { - api: "responses", - model: "claude-test", - providerId: "anthropic", - providerName: "anthropic", - }, - response: { - output: { content: [{ text: "done", type: "text" }] }, - stopReason: "stop", - usage, - }, - runId: "run-1", - turnId: "turn-1", - type: "turn", - v: 3, - }); - emit({ - durationMs: 50, - eventIndex: 6, - isError: false, - operationId: "op-1", - operationKind: "prompt", - result: { text: "PROMPT_DONE", usage }, - runId: "run-1", - type: "operation", - usage, - v: 3, - }); - - const workflowSpan = findSpan("workflow:research"); - const turnSpan = findSpan("flue.turn"); - const toolSpan = findSpan("tool:lookup"); - const operationSpan = findSpan("flue.prompt"); - - expect(workflowSpan?.args.event.input).toMatchObject({ - metadata: { scenario: "flue-v1" }, - topic: "native instrumentation", - }); - expect(turnSpan?.args.event).toMatchObject({ - input: [{ content: "Find native hooks", role: "user" }], - metadata: { - "flue.api": "responses", - "flue.model": "claude-test", - "flue.provider": "anthropic", - "flue.system_prompt": "Be exact", - provider: "anthropic", - reasoning: "medium", - tools: [{ name: "lookup" }], - }, - }); - expect(toolSpan?.log).toHaveBeenCalledWith( - expect.objectContaining({ output: { ok: true } }), - ); - expect(turnSpan?.log).toHaveBeenCalledWith( - expect.objectContaining({ - metadata: expect.objectContaining({ - "flue.stop_reason": "stop", - }), - output: { content: [{ text: "done", type: "text" }] }, - }), - ); - expect(operationSpan?.log).toHaveBeenCalledWith( - expect.objectContaining({ output: "PROMPT_DONE" }), - ); - }); - it("maps released Flue 2.0 observations and retains the final agent output", () => { const emit = observeEvents(); const usage = flueUsage(); @@ -790,59 +540,65 @@ describe("Flue observe instrumentation", () => { type: "operation_start", }); emit({ - input: { - messages: [previousUser, previousAssistant, currentUser], - systemPrompt: "Be precise", - tools: [{ name: "lookup" }], - }, operationId: "op-1", purpose: "agent", + request: { + input: { + messages: [previousUser, previousAssistant, currentUser], + systemPrompt: "Be precise", + tools: [{ name: "lookup" }], + }, + }, runId: "run-1", turnId: "turn-1", type: "turn_request", }); emit({ operationId: "op-1", - output: toolCall, purpose: "agent", + response: { output: toolCall }, runId: "run-1", turnId: "turn-1", type: "turn", }); emit({ - input: { - messages: [ - previousUser, - previousAssistant, - currentUser, - toolCall, - toolResult, - ], - systemPrompt: "Be precise", - tools: [{ name: "lookup" }], - }, operationId: "op-1", purpose: "agent", + request: { + input: { + messages: [ + previousUser, + previousAssistant, + currentUser, + toolCall, + toolResult, + ], + systemPrompt: "Be precise", + tools: [{ name: "lookup" }], + }, + }, runId: "run-1", turnId: "turn-1", type: "turn_request", }); emit({ operationId: "op-1", - output: { content: "done", role: "assistant" }, purpose: "agent", + response: { output: { content: "done", role: "assistant" } }, runId: "run-1", turnId: "turn-1", type: "turn", }); emit({ - input: { - messages: [currentUser], - systemPrompt: "Use compacted context", - tools: [{ name: "search" }], - }, operationId: "op-1", purpose: "agent", + request: { + input: { + messages: [currentUser], + systemPrompt: "Use compacted context", + tools: [{ name: "search" }], + }, + }, runId: "run-1", turnId: "turn-1", type: "turn_request", @@ -927,13 +683,15 @@ describe("Flue observe instrumentation", () => { type: "operation_start", }); emit({ - input: { - messages: [{ content: "finish", role: "user" }], - }, - model: "claude-test", operationId: "op-1", - provider: "anthropic", purpose: "agent", + request: { + input: { + messages: [{ content: "finish", role: "user" }], + }, + providerName: "anthropic", + requestedModel: "claude-test", + }, runId: "run-1", turnId: "turn-1", type: "turn_request", @@ -1008,13 +766,15 @@ describe("Flue observe instrumentation", () => { type: "compaction_start", }); emit({ - input: { - messages: [{ content: "summarize", role: "user" }], - }, - model: "gpt-test", operationId: "op-compact", - provider: "openai", purpose: "compaction_prefix", + request: { + input: { + messages: [{ content: "summarize", role: "user" }], + }, + providerName: "openai", + requestedModel: "gpt-test", + }, runId: "run-1", session: "main", turnId: "turn-compact", @@ -1220,7 +980,7 @@ describe("Flue observe instrumentation", () => { runId: "run-1", toolCallId: "tool-1", toolName: "lookup", - type: "tool_call", + type: "tool", }); expect(mockCurrentParentSpan.current).toBeUndefined(); @@ -1236,10 +996,10 @@ describe("Flue observe instrumentation", () => { type: "run_start", workflowName: "research", }, - { id: "ctx-1", runId: "run-1" }, + { id: "ctx-1" }, ); const workflowSpan = findSpan("workflow:research"); - const appSpan = await braintrustFlueObserver.interceptor( + const appSpan = await braintrustFlueInstrumentation().interceptor( { phase: "start", runId: "run-1", @@ -1247,7 +1007,7 @@ describe("Flue observe instrumentation", () => { type: "workflow", workflowName: "research", }, - { eventContext: { id: "ctx-1", runId: "run-1" } }, + { eventContext: { id: "ctx-1" } }, async () => mockStartSpan({ name: "app.phase" }), ); @@ -1283,7 +1043,7 @@ describe("Flue observe instrumentation", () => { }); const toolSpan = findSpan("tool:lookup"); - const appSpan = await braintrustFlueObserver.interceptor( + const appSpan = await braintrustFlueInstrumentation().interceptor( { toolCallId: "tool-1", toolName: "lookup", type: "tool" }, { operationId: "op-prompt", runId: "run-1" }, async () => mockStartSpan({ name: "app.tool-phase" }), @@ -1312,11 +1072,13 @@ describe("Flue observe instrumentation", () => { type: "operation_start", }); emit({ - input: { messages: [{ content: "hello", role: "user" }] }, - model: "claude-test", operationId: "op-prompt", - provider: "anthropic", purpose: "agent", + request: { + input: { messages: [{ content: "hello", role: "user" }] }, + providerName: "anthropic", + requestedModel: "claude-test", + }, runId: "run-1", turnId: "turn-1", type: "turn_request", @@ -1332,17 +1094,17 @@ describe("Flue observe instrumentation", () => { const operationSpan = findSpan("flue.prompt"); const turnSpan = findSpan("flue.turn"); const taskSpan = findSpan("flue.task"); - const agentAppSpan = await braintrustFlueObserver.interceptor( + const agentAppSpan = await braintrustFlueInstrumentation().interceptor( { operationId: "op-prompt", operationKind: "prompt", type: "agent" }, { operationId: "op-prompt", runId: "run-1" }, async () => mockStartSpan({ name: "app.agent-phase" }), ); - const modelAppSpan = await braintrustFlueObserver.interceptor( + const modelAppSpan = await braintrustFlueInstrumentation().interceptor( { turnId: "turn-1", type: "model" }, { operationId: "op-prompt", runId: "run-1", turnId: "turn-1" }, async () => mockStartSpan({ name: "app.model-phase" }), ); - const taskAppSpan = await braintrustFlueObserver.interceptor( + const taskAppSpan = await braintrustFlueInstrumentation().interceptor( { taskId: "task-1", type: "task" }, { operationId: "op-prompt", runId: "run-1", taskId: "task-1" }, async () => mockStartSpan({ name: "app.task-phase" }), @@ -1354,149 +1116,9 @@ describe("Flue observe instrumentation", () => { expect(mockCurrentParentSpan.current).toBeUndefined(); }); - it("subscribes transformed Flue contexts for auto instrumentation", () => { - const plugin = new FluePlugin(); - const contextSubscribers: Array<(event: unknown) => unknown> = []; - const unsubscribeContext = vi.fn(); - const context = { - id: "ctx-1", - runId: "run-1", - subscribeEvent: vi.fn((subscriber: (event: unknown) => unknown) => { - contextSubscribers.push(subscriber); - return unsubscribeContext; - }), - }; - - plugin.enable(); - expect(mockNewTracingChannel).toHaveBeenCalledWith( - CREATE_CONTEXT_CHANNEL_NAME, - ); - expect( - tracingChannel(CREATE_CONTEXT_CHANNEL_NAME).subscribe, - ).toHaveBeenCalledTimes(1); - - emitCreateContextEnd(context); - - expect(context.subscribeEvent).toHaveBeenCalledTimes(1); - contextSubscribers[0]?.({ - runId: "run-1", - type: "run_start", - workflowName: "auto-research", - }); - - expect( - findSpan("workflow:auto-research")?.args.event.metadata, - ).toMatchObject({ - "flue.context_id": "ctx-1", - "flue.context_run_id": "run-1", - }); - contextSubscribers[0]?.({ - durationMs: 1, - isError: false, - result: "done", - runId: "run-1", - type: "run_end", - }); - expect(unsubscribeContext).toHaveBeenCalledTimes(1); - - plugin.disable(); - - expect( - tracingChannel(CREATE_CONTEXT_CHANNEL_NAME).unsubscribe, - ).toHaveBeenCalledTimes(1); - expect(unsubscribeContext).toHaveBeenCalledTimes(1); - }); - - it("keeps auto instrumentation idempotent across plugin instances", () => { - const first = new FluePlugin(); - const second = new FluePlugin(); - const contextSubscribers: Array<(event: unknown) => unknown> = []; - const unsubscribeContext = vi.fn(); - const context = { - subscribeEvent: vi.fn((subscriber: (event: unknown) => unknown) => { - contextSubscribers.push(subscriber); - return unsubscribeContext; - }), - }; - - first.enable(); - second.enable(); - emitCreateContextEnd(context); - emitCreateContextEnd(context); - - expect( - tracingChannel(CREATE_CONTEXT_CHANNEL_NAME).subscribe, - ).toHaveBeenCalledTimes(1); - expect(context.subscribeEvent).toHaveBeenCalledTimes(1); - - first.disable(); - expect( - tracingChannel(CREATE_CONTEXT_CHANNEL_NAME).unsubscribe, - ).not.toHaveBeenCalled(); - expect(unsubscribeContext).not.toHaveBeenCalled(); - - second.disable(); - expect( - tracingChannel(CREATE_CONTEXT_CHANNEL_NAME).unsubscribe, - ).toHaveBeenCalledTimes(1); - - contextSubscribers[0]?.({ - runId: "run-after-disable", - type: "run_start", - workflowName: "after-disable", - }); - expect(unsubscribeContext).toHaveBeenCalledTimes(1); - expect(findSpan("workflow:after-disable")).toBeUndefined(); - }); - - it("unsubscribes direct Flue contexts on terminal operation events", () => { - const plugin = new FluePlugin(); - const contextSubscribers: Array<(event: unknown) => unknown> = []; - const unsubscribeContext = vi.fn(); - const context = { - id: "direct-agent-1", - subscribeEvent: vi.fn((subscriber: (event: unknown) => unknown) => { - contextSubscribers.push(subscriber); - return unsubscribeContext; - }), - }; - - plugin.enable(); - emitCreateContextEnd(context); - contextSubscribers[0]?.({ - durationMs: 2, - instanceId: "direct-agent-1", - isError: false, - operationId: "op-1", - operationKind: "prompt", - result: "done", - type: "operation", - }); - - expect(unsubscribeContext).toHaveBeenCalledTimes(1); - expect(findSpan("flue.prompt")).toBeDefined(); - - plugin.disable(); - }); - function observeEvents() { return (event: unknown, ctx?: unknown) => - braintrustFlueObserver(event, ctx); - } - - function emitCreateContextEnd(result: unknown) { - for (const handlers of tracingChannel(CREATE_CONTEXT_CHANNEL_NAME) - .__handlers) { - handlers.end?.({ result }); - } - } - - function tracingChannel(channelName: string) { - const channel = mockTracingChannels.get(channelName); - if (!channel) { - throw new Error(`Missing mocked tracing channel: ${channelName}`); - } - return channel; + braintrustFlueInstrumentation().observe(event, ctx); } function findSpan(name: string) { diff --git a/js/src/instrumentation/plugins/flue-plugin.ts b/js/src/instrumentation/plugins/flue-plugin.ts index 81de77992..1426afde6 100644 --- a/js/src/instrumentation/plugins/flue-plugin.ts +++ b/js/src/instrumentation/plugins/flue-plugin.ts @@ -1,7 +1,5 @@ -import { BasePlugin, toLoggedError } from "../core"; +import { toLoggedError } from "../core"; import { debugLogger } from "../../debug-logger"; -import type { ChannelMessage } from "../core/channel-definitions"; -import type { IsoChannelHandlers } from "../../isomorph"; import { BRAINTRUST_CURRENT_SPAN_STORE, NOOP_SPAN, @@ -11,13 +9,11 @@ import { withCurrent, } from "../../logger"; import type { Span, StartSpanArgs } from "../../logger"; -import type { CurrentSpanStore } from "../../logger"; import { INSTRUMENTATION_NAMES, withSpanInstrumentationName, } from "../../span-origin"; import { SpanTypeAttribute } from "../../../util/index"; -import { flueChannels } from "./flue-channels"; import type { FlueBaseEvent, FlueCompactionEvent, @@ -28,7 +24,6 @@ import type { FlueExecutionInterceptor, FlueExecutionOperation, FlueInstrumentation, - FlueObservableContext, FlueOperationEvent, FlueOperationKind, FlueOperationStartEvent, @@ -44,20 +39,6 @@ import type { FlueTurnRequestEvent, } from "../../vendor-sdk-types/flue"; -type FlueObserver = (event: unknown, ctx?: unknown) => void; -type BraintrustFlueObserver = FlueObserver & FlueInstrumentation; - -type FlueAutoState = { - createContextChannel?: ReturnType< - typeof flueChannels.createContext.tracingChannel - >; - createContextHandlers?: IsoChannelHandlers< - ChannelMessage - >; - contexts: WeakSet; - refCount: number; -}; - type SpanState = { latestAgentOutput?: unknown; loggedInput?: boolean; @@ -73,7 +54,6 @@ type FlueTurnInputState = { toolsFingerprint?: string; }; -const FLUE_AUTO_STATE = Symbol.for("braintrust.flue.auto-state"); const FLUE_INSTRUMENTATION_KEY = Symbol.for("braintrust.flue.instrumentation"); const FLUE_OBSERVE_BRIDGE = Symbol.for("braintrust.flue.observe-bridge"); @@ -88,9 +68,9 @@ const interceptFlueExecution: FlueExecutionInterceptor = ( ) => getObserveBridge().intercept(operation, ctx, next); /** - * Manual instrumentation for flue. + * Instrumentation for flue. * - * This should be passed to flue's `instrument()` API if not using auto-instrumentation: `instrument(braintrustFlueInstrumentation())` + * Pass this to flue's `instrument()` API: `instrument(braintrustFlueInstrumentation())` */ export function braintrustFlueInstrumentation(): FlueInstrumentation { return { @@ -101,71 +81,6 @@ export function braintrustFlueInstrumentation(): FlueInstrumentation { }; } -/** - * Observer for flue pre version 1.0.0. - * - * This observer should be passed to flue's `observe()` API if not using auto-instrumentation. - */ -export const braintrustFlueObserver: BraintrustFlueObserver = Object.assign( - observeFlue, - braintrustFlueInstrumentation(), -); - -export class FluePlugin extends BasePlugin { - protected onEnable(): void { - this.unsubscribers.push(enableFlueAutoInstrumentation()); - } - - protected onDisable(): void { - for (const unsubscribe of this.unsubscribers) { - unsubscribe(); - } - this.unsubscribers = []; - } -} - -function enableFlueAutoInstrumentation(): () => void { - const state = getAutoState(); - state.refCount += 1; - - if (!state.createContextHandlers) { - const createContextChannel = flueChannels.createContext.tracingChannel(); - const handlers: IsoChannelHandlers< - ChannelMessage - > = { - end: (event) => { - subscribeToFlueContext(event.result, state); - }, - }; - - createContextChannel.subscribe(handlers); - state.createContextChannel = createContextChannel; - state.createContextHandlers = handlers; - } - - let released = false; - return () => { - if (released) { - return; - } - released = true; - releaseAutoState(state); - }; -} - -function getAutoState(): FlueAutoState { - const existing = Reflect.get(globalThis, FLUE_AUTO_STATE); - if (isAutoState(existing)) { - return existing; - } - const state: FlueAutoState = { - contexts: new WeakSet(), - refCount: 0, - }; - Reflect.set(globalThis, FLUE_AUTO_STATE, state); - return state; -} - function getObserveBridge(): FlueObserveBridge { const existing = Reflect.get(globalThis, FLUE_OBSERVE_BRIDGE); if (isFlueObserveBridge(existing)) { @@ -180,94 +95,7 @@ function isFlueObserveBridge(value: unknown): value is FlueObserveBridge { return ( isObjectLike(value) && typeof Reflect.get(value, "handle") === "function" && - typeof Reflect.get(value, "reset") === "function" - ); -} - -function isAutoState(value: unknown): value is FlueAutoState { - return ( - isObjectLike(value) && - Reflect.get(value, "contexts") instanceof WeakSet && - typeof Reflect.get(value, "refCount") === "number" - ); -} - -function releaseAutoState(state: FlueAutoState): void { - state.refCount -= 1; - if (state.refCount > 0) { - return; - } - - try { - if (state.createContextChannel && state.createContextHandlers) { - state.createContextChannel.unsubscribe(state.createContextHandlers); - } - } finally { - Reflect.deleteProperty(globalThis, FLUE_AUTO_STATE); - } -} - -function subscribeToFlueContext(value: unknown, state: FlueAutoState): void { - if (!isObservableFlueContext(value) || state.contexts.has(value)) { - return; - } - - const ctx = flueContextFromUnknown(value); - let released = false; - let unsubscribe: (() => void) | undefined; - const release = () => { - if (released) { - return; - } - released = true; - try { - unsubscribe?.(); - } catch (error) { - logInstrumentationError("Flue context unsubscribe", error); - } - }; - - try { - unsubscribe = value.subscribeEvent((event) => { - if (state.refCount <= 0) { - release(); - return; - } - - braintrustFlueObserver(event, ctx); - if (isAutoContextTerminalEvent(event, ctx)) { - release(); - } - }); - state.contexts.add(value); - } catch (error) { - logInstrumentationError("Flue context subscription", error); - } -} - -function isAutoContextTerminalEvent( - event: unknown, - ctx: FlueContext | undefined, -): boolean { - if (!isObjectLike(event)) { - return false; - } - const type = Reflect.get(event, "type"); - if (type === "run_end") { - return true; - } - if (type !== "operation") { - return false; - } - return !ctx?.runId && typeof Reflect.get(event, "runId") !== "string"; -} - -function isObservableFlueContext( - value: unknown, -): value is FlueObservableContext { - return ( - isObjectLike(value) && - typeof Reflect.get(value, "subscribeEvent") === "function" + typeof Reflect.get(value, "intercept") === "function" ); } @@ -282,7 +110,6 @@ function isFlueEvent(event: object): event is FlueEvent { type === "turn_request" || type === "turn" || type === "tool_start" || - type === "tool_call" || type === "tool" || type === "task_start" || type === "task" || @@ -296,11 +123,7 @@ function flueContextFromUnknown(ctx: unknown): FlueContext | undefined { return undefined; } const id = Reflect.get(ctx, "id"); - const runId = Reflect.get(ctx, "runId"); - return { - ...(typeof id === "string" ? { id } : {}), - ...(typeof runId === "string" ? { runId } : {}), - }; + return typeof id === "string" ? { id } : undefined; } function isObjectLike(value: unknown): value is object { @@ -332,16 +155,6 @@ class FlueObserveBridge { } } - reset(): void { - this.compactionsByKey.clear(); - this.operationsById.clear(); - this.runsById.clear(); - this.seenEvents = new WeakSet(); - this.tasksById.clear(); - this.toolsByKey.clear(); - this.turnsByKey.clear(); - } - intercept( operation: FlueExecutionOperation, executionContext: FlueExecutionContext, @@ -389,7 +202,6 @@ class FlueObserveBridge { flueContextFromUnknown(executionContext.eventContext) ?? flueContextFromUnknown({ id: operation.workflowName, - runId: operation.runId, }); if (operation.phase === "resume") { this.handleRunResume( @@ -399,7 +211,6 @@ class FlueObserveBridge { startedAt: operation.startedAt, timestamp: new Date().toISOString(), type: "run_resume", - v: 3, workflowName: operation.workflowName, }, ctx, @@ -413,7 +224,6 @@ class FlueObserveBridge { startedAt: operation.startedAt, timestamp: new Date().toISOString(), type: "run_start", - v: 3, workflowName: operation.workflowName, }, ctx, @@ -472,7 +282,6 @@ class FlueObserveBridge { case "tool_start": this.handleToolStart(event); return; - case "tool_call": case "tool": this.handleToolCall(event); return; @@ -501,15 +310,12 @@ class FlueObserveBridge { return; } - const workflowName = - event.workflowName ?? - event.owner?.workflowName ?? - (typeof ctx?.id === "string" ? ctx.id : "unknown"); - const input = flueRunInput(event); + const workflowName = event.workflowName ?? ctx?.id ?? "unknown"; + const input = event.input; const metadata = { ...extractPayloadMetadata(input), ...extractEventMetadata(event, ctx), - ...(workflowName ? { "flue.workflow_name": workflowName } : {}), + "flue.workflow_name": workflowName, provider: "flue", }; const existing = this.runsById.get(event.runId); @@ -543,11 +349,10 @@ class FlueObserveBridge { return; } - const workflowName = - event.workflowName ?? (typeof ctx?.id === "string" ? ctx.id : "unknown"); + const workflowName = event.workflowName ?? ctx?.id ?? "unknown"; const metadata = { ...extractEventMetadata(event, ctx), - ...(workflowName ? { "flue.workflow_name": workflowName } : {}), + "flue.workflow_name": workflowName, "flue.workflow_phase": "resume", provider: "flue", }; @@ -664,20 +469,20 @@ class FlueObserveBridge { } private handleTurnRequest(event: FlueTurnRequestEvent): void { - const key = turnKey(event); + const key = event.turnId; if (!key) { return; } - const input = flueTurnRequestInput(event); + const input = event.request?.input; const operation = event.operationId ? this.operationsById.get(event.operationId) : undefined; const turnInput = prepareFlueTurnInput(event, input, operation); - const model = flueTurnRequestModel(event); - const provider = flueTurnRequestProvider(event); - const api = flueTurnRequestApi(event); - const reasoning = flueTurnRequestReasoning(event); + const model = event.request?.requestedModel; + const provider = event.request?.providerName ?? event.request?.providerId; + const api = event.request?.api; + const reasoning = event.request?.reasoningLevel; const metadata = { ...extractEventMetadata(event), ...(api ? { "flue.api": api } : {}), @@ -707,19 +512,20 @@ class FlueObserveBridge { } private handleTurn(event: FlueTurnEvent): void { - const key = turnKey(event); + const key = event.turnId; if (!key) { return; } const state = this.turnsByKey.get(key) ?? this.startSyntheticTurn(event); - const model = flueTurnModel(event); - const provider = flueTurnProvider(event); - const api = flueTurnApi(event); - const stopReason = flueTurnStopReason(event); - const usage = flueTurnUsage(event); - const output = flueTurnOutput(event); - const error = flueTurnError(event); + const model = + event.response?.responseModel ?? event.request?.requestedModel; + const provider = event.request?.providerName ?? event.request?.providerId; + const api = event.request?.api; + const stopReason = event.response?.finishReason; + const usage = event.response?.usage; + const output = event.response?.output; + const error = event.response?.error; const metadata = { ...state.metadata, ...extractEventMetadata(event), @@ -758,7 +564,7 @@ class FlueObserveBridge { return; } - const input = flueToolInput(event); + const input = event.args; const metadata = { ...extractEventMetadata(event), ...(event.toolName ? { "flue.tool_name": event.toolName } : {}), @@ -933,7 +739,7 @@ class FlueObserveBridge { } private parentSpanForEvent(event: FlueBaseEvent): Span | undefined { - const turn = turnKey(event); + const turn = event.turnId; if (turn) { const turnState = this.turnsByKey.get(turn); if (turnState) { @@ -1017,9 +823,10 @@ class FlueObserveBridge { } private startSyntheticTurn(event: FlueTurnEvent): SpanState { - const model = flueTurnModel(event); - const provider = flueTurnProvider(event); - const api = flueTurnApi(event); + const model = + event.response?.responseModel ?? event.request?.requestedModel; + const provider = event.request?.providerName ?? event.request?.providerId; + const api = event.request?.api; const metadata = { ...extractEventMetadata(event), ...(api ? { "flue.api": api } : {}), @@ -1208,10 +1015,7 @@ function extractEventMetadata( ...(event.taskId ? { "flue.task_id": event.taskId } : {}), ...(event.operationId ? { "flue.operation_id": event.operationId } : {}), ...(event.turnId ? { "flue.turn_id": event.turnId } : {}), - ...(typeof ctx?.id === "string" ? { "flue.context_id": ctx.id } : {}), - ...(typeof ctx?.runId === "string" - ? { "flue.context_run_id": ctx.runId } - : {}), + ...(ctx ? { "flue.context_id": ctx.id } : {}), }; } @@ -1226,19 +1030,9 @@ function extractPayloadMetadata(payload: unknown): Record { return Object.fromEntries(Object.entries(metadata)); } -function flueRunInput(event: FlueRunStartEvent): unknown { - return event.input !== undefined ? event.input : event.payload; -} - -function flueTurnRequestInput( - event: FlueTurnRequestEvent, -): FlueTurnRequestEvent["input"] { - return event.request?.input ?? event.input; -} - function prepareFlueTurnInput( event: FlueTurnRequestEvent, - input: FlueTurnRequestEvent["input"], + input: NonNullable["input"], operation: SpanState | undefined, ): { messages: unknown[] | undefined; metadata: Record } { const messages = input?.messages; @@ -1347,90 +1141,15 @@ function flueOperationInput(event: FlueOperationEvent): unknown { : undefined; } -function flueTurnRequestModel(event: FlueTurnRequestEvent): string | undefined { - return event.request?.requestedModel ?? event.request?.model ?? event.model; -} - -function flueTurnRequestProvider( - event: FlueTurnRequestEvent, -): string | undefined { - return ( - event.request?.providerName ?? event.provider ?? event.request?.providerId - ); -} - -function flueTurnRequestApi(event: FlueTurnRequestEvent): string | undefined { - return event.request?.api ?? event.api; -} - -function flueTurnRequestReasoning( - event: FlueTurnRequestEvent, -): string | undefined { - return ( - event.request?.reasoningLevel ?? event.request?.reasoning ?? event.reasoning - ); -} - -function flueTurnModel(event: FlueTurnEvent): string | undefined { - return ( - event.response?.responseModel ?? - event.request?.requestedModel ?? - event.request?.model ?? - event.model - ); -} - -function flueTurnProvider(event: FlueTurnEvent): string | undefined { - return ( - event.request?.providerName ?? event.provider ?? event.request?.providerId - ); -} - -function flueTurnApi(event: FlueTurnEvent): string | undefined { - return event.request?.api ?? event.api; -} - -function flueTurnUsage(event: FlueTurnEvent): unknown { - return event.response?.usage ?? event.usage; -} - -function flueTurnOutput(event: FlueTurnEvent): unknown { - return event.response?.output ?? event.output; -} - -function flueTurnStopReason(event: FlueTurnEvent): string | undefined { - return ( - event.response?.finishReason ?? - event.response?.stopReason ?? - event.stopReason - ); -} - -function flueTurnError(event: FlueTurnEvent): unknown { - return ( - event.response?.error ?? event.response?.errorInfo?.message ?? event.error - ); -} - -function flueToolInput(event: FlueToolStartEvent): unknown { - if (event.args !== undefined) { - return event.args; - } - if (event.arguments !== undefined) { - return event.arguments; - } - return event.input; -} - function flueToolOutput(event: FlueToolCallEvent): unknown { if (Object.hasOwn(event, "effectiveResult")) { return event.effectiveResult; } - return event.output !== undefined ? event.output : event.result; + return event.result; } function flueToolError(event: FlueToolCallEvent): unknown { - return event.error ?? event.errorInfo?.message ?? flueToolOutput(event); + return event.errorInfo?.message ?? flueToolOutput(event); } function operationOutput(event: FlueOperationEvent): unknown { @@ -1529,10 +1248,6 @@ function eventTime(value: unknown): number | undefined { return Number.isFinite(timestamp) ? timestamp / 1000 : undefined; } -function turnKey(event: FlueBaseEvent): string | undefined { - return event.turnId; -} - function toolKey(event: FlueBaseEvent & { toolCallId?: string }): string { return `${event.turnId ?? event.operationId ?? event.taskId ?? event.runId ?? "unknown"}:${event.toolCallId ?? "unknown"}`; } @@ -1594,11 +1309,7 @@ function runWithCurrentSpanStore( const state = _internalGetGlobalState(); const contextManager = state?.contextManager; const currentSpanStore = contextManager - ? ( - contextManager as { - [BRAINTRUST_CURRENT_SPAN_STORE]?: CurrentSpanStore; - } - )[BRAINTRUST_CURRENT_SPAN_STORE] + ? Reflect.get(contextManager, BRAINTRUST_CURRENT_SPAN_STORE) : undefined; if (contextManager && typeof currentSpanStore?.run === "function") { diff --git a/js/src/instrumentation/plugins/instrumentation-names.test.ts b/js/src/instrumentation/plugins/instrumentation-names.test.ts index d8f4418b8..5b0585839 100644 --- a/js/src/instrumentation/plugins/instrumentation-names.test.ts +++ b/js/src/instrumentation/plugins/instrumentation-names.test.ts @@ -14,7 +14,6 @@ import { cloudflareAgentsChannels } from "./cloudflare-agents-channels"; import { cloudflareThinkChannels } from "./cloudflare-think-channels"; import { cohereChannels } from "./cohere-channels"; import { cursorSDKChannels } from "./cursor-sdk-channels"; -import { flueChannels } from "./flue-channels"; import { genkitChannels, genkitCoreChannels } from "./genkit-channels"; import { gitHubCopilotChannels } from "./github-copilot-channels"; import { googleADKChannels } from "./google-adk-channels"; @@ -56,7 +55,6 @@ describe("built-in instrumentation provenance names", () => { ], [cohereChannels.chat, INSTRUMENTATION_NAMES.COHERE], [cursorSDKChannels.create, INSTRUMENTATION_NAMES.CURSOR_SDK], - [flueChannels.createContext, INSTRUMENTATION_NAMES.FLUE], [genkitChannels.generate, INSTRUMENTATION_NAMES.GENKIT], [genkitCoreChannels.actionSpan, INSTRUMENTATION_NAMES.GENKIT], [gitHubCopilotChannels.createSession, INSTRUMENTATION_NAMES.GITHUB_COPILOT], diff --git a/js/src/vendor-sdk-types/flue.ts b/js/src/vendor-sdk-types/flue.ts index 367046fe2..2f7c75222 100644 --- a/js/src/vendor-sdk-types/flue.ts +++ b/js/src/vendor-sdk-types/flue.ts @@ -1,5 +1,5 @@ /** - * Vendored types for @flue/runtime observe/instrument-based instrumentation. + * Vendored types for @flue/runtime instrumentation. * * Keep this surface intentionally narrow. These types are not exported to SDK * users and should only cover fields we read, correlate, or log. @@ -16,17 +16,11 @@ export interface FlueUsage { cacheWrite?: number; totalTokens?: number; cost?: { - input?: number; - output?: number; - cacheRead?: number; - cacheWrite?: number; total?: number; }; } export interface FlueBaseEvent { - type?: string; - v?: number; runId?: string; instanceId?: string; submissionId?: string; @@ -41,7 +35,6 @@ export interface FlueBaseEvent { harness?: string; operationId?: string; turnId?: string; - [key: string]: unknown; } export interface FlueRunStartEvent extends FlueBaseEvent { @@ -49,13 +42,7 @@ export interface FlueRunStartEvent extends FlueBaseEvent { runId: string; startedAt?: string; workflowName?: string; - owner?: { - kind?: string; - workflowName?: string; - instanceId?: string; - }; input?: unknown; - payload?: unknown; } export interface FlueRunResumeEvent extends FlueBaseEvent { @@ -91,10 +78,9 @@ export interface FlueOperationEvent extends FlueBaseEvent { result?: unknown; agentInput?: { text?: string; - images?: Array<{ mimeType?: string }>; }; agentOutput?: - | { type?: "text"; text?: string; finishReason?: string } + | { type?: "text"; text?: string } | { type?: "data"; data?: unknown }; usage?: FlueUsage; } @@ -103,14 +89,6 @@ export interface FlueTurnRequestEvent extends FlueBaseEvent { type: "turn_request"; turnId: string; purpose: FlueTurnPurpose; - model?: string; - provider?: string; - api?: string; - input?: { - systemPrompt?: string; - messages?: unknown[]; - tools?: unknown[]; - }; request?: { api?: string; input?: { @@ -118,14 +96,11 @@ export interface FlueTurnRequestEvent extends FlueBaseEvent { messages?: unknown[]; tools?: unknown[]; }; - model?: string; requestedModel?: string; providerId?: string; providerName?: string; - reasoning?: string; reasoningLevel?: string; }; - reasoning?: string; } export interface FlueTurnEvent extends FlueBaseEvent { @@ -133,12 +108,8 @@ export interface FlueTurnEvent extends FlueBaseEvent { turnId: string; purpose?: FlueTurnPurpose; durationMs?: number; - model?: string; - provider?: string; - api?: string; request?: { api?: string; - model?: string; requestedModel?: string; providerId?: string; providerName?: string; @@ -146,17 +117,11 @@ export interface FlueTurnEvent extends FlueBaseEvent { response?: { output?: unknown; usage?: FlueUsage; - stopReason?: string; finishReason?: string; responseModel?: string; error?: unknown; - errorInfo?: { type?: string; message?: string }; }; - output?: unknown; - usage?: FlueUsage; - stopReason?: string; isError?: boolean; - error?: unknown; } export interface FlueToolStartEvent extends FlueBaseEvent { @@ -164,21 +129,15 @@ export interface FlueToolStartEvent extends FlueBaseEvent { toolName?: string; toolCallId: string; args?: unknown; - arguments?: unknown; - input?: unknown; - origin?: "model" | "caller" | "framework" | "adapter"; - description?: string; } export interface FlueToolCallEvent extends FlueBaseEvent { - type: "tool_call" | "tool"; + type: "tool"; toolName?: string; toolCallId: string; isError?: boolean; result?: unknown; effectiveResult?: unknown; - output?: unknown; - error?: unknown; errorInfo?: { type?: string; message?: string }; durationMs?: number; } @@ -197,9 +156,6 @@ export interface FlueTaskEvent extends FlueBaseEvent { agent?: string; isError?: boolean; result?: unknown; - agentOutput?: - | { type?: "text"; text?: string; finishReason?: string } - | { type?: "data"; data?: unknown }; errorInfo?: { type?: string; message?: string }; durationMs?: number; } @@ -222,12 +178,7 @@ export interface FlueCompactionEvent extends FlueBaseEvent { } export interface FlueContext { - readonly id?: string; - readonly runId?: string; -} - -export interface FlueObservableContext extends FlueContext { - subscribeEvent(callback: (event: unknown) => unknown): () => void; + readonly id: string; } export type FlueExecutionOperation = @@ -244,25 +195,12 @@ export type FlueExecutionOperation = | { type: "task"; taskId: string } | { type: "coordinator"; phase: "reconcile" }; -export interface FlueTraceCarrier { - traceparent: string; - tracestate?: string; -} - export interface FlueExecutionContext { eventContext?: FlueContext; runId?: string; - instanceId?: string; - submissionId?: string; - dispatchId?: string; - agentName?: string; - conversationId?: string; - harness?: string; - session?: string; operationId?: string; turnId?: string; taskId?: string; - traceCarrier?: FlueTraceCarrier; } export type FlueExecutionInterceptor = ( From 5aa890d8a5dee9ab840bbcacadc75933c2cd5f24 Mon Sep 17 00:00:00 2001 From: Luca Forstner Date: Wed, 19 Aug 2026 17:43:44 +0200 Subject: [PATCH 2/8] feat(v4)!: Add Node Engine to package json (#2378) ref [braintrustdata/braintrust-sdk-javascript#1841](https://github.com/braintrustdata/braintrust-sdk-javascript/issues/1841) matches what we have documented in readme --- .changeset/add-node-engine.md | 5 +++++ js/package.json | 3 +++ 2 files changed, 8 insertions(+) create mode 100644 .changeset/add-node-engine.md diff --git a/.changeset/add-node-engine.md b/.changeset/add-node-engine.md new file mode 100644 index 000000000..10744b77a --- /dev/null +++ b/.changeset/add-node-engine.md @@ -0,0 +1,5 @@ +--- +"braintrust": major +--- + +feat(v4)!: Add Node Engine to package json diff --git a/js/package.json b/js/package.json index 2d0ca836e..3841c8b24 100644 --- a/js/package.json +++ b/js/package.json @@ -174,6 +174,9 @@ }, "author": "", "license": "MIT", + "engines": { + "node": "^20.12.0 || >=22.13.0" + }, "devDependencies": { "@anthropic-ai/sdk": "^0.60.0", "@nodelib/fs.walk": "^1.2.8", From ded0424fb1badc899f70cf7450ea7785beceac03 Mon Sep 17 00:00:00 2001 From: Luca Forstner Date: Wed, 19 Aug 2026 18:43:45 +0200 Subject: [PATCH 3/8] ref(v4)!: Remove JS Braintrust CLI (#2377) Ref [braintrustdata/braintrust-sdk-javascript#1841](https://github.com/braintrustdata/braintrust-sdk-javascript/issues/1841) `bt` is supposed to be used over the JS CLI --- .changeset/remove-js-braintrust-cli.md | 5 + .github/workflows/checks.yaml | 16 +- README.md | 2 +- integrations/otel-js/src/otel-compat.test.ts | 20 +- js/dev/authorize.ts | 217 --- js/dev/errorHandler.ts | 37 - js/dev/index.ts | 5 - js/dev/server.test.ts | 21 - js/dev/server.ts | 407 ------ js/dev/stream.ts | 14 - js/dev/types.ts | 123 -- js/eslint.config.ts | 49 +- js/package.json | 34 +- js/scripts/test-output.ts | 194 --- .../shared/src/suites/import-verification.ts | 26 +- js/src/cli/auto-instrumentation.test.ts | 306 ----- js/src/cli/debug-logging.test.ts | 71 - js/src/cli/functions/infer-source.ts | 212 --- js/src/cli/functions/load-module.ts | 54 - js/src/cli/functions/upload.test.ts | 119 -- js/src/cli/functions/upload.ts | 516 ------- js/src/cli/index.ts | 1194 ----------------- js/src/cli/integration.test.ts | 285 ---- js/src/cli/jest/nodeModulesPaths.ts | 74 - js/src/cli/progress.ts | 1 - js/src/cli/reporters/eval.ts | 227 ---- js/src/cli/reporters/progress.ts | 51 - js/src/cli/types.ts | 40 - js/src/cli/util/bundle.ts | 91 -- js/src/cli/util/debug-logging.test.ts | 53 - js/src/cli/util/debug-logging.ts | 37 - .../cli/util/external-packages-plugin.test.ts | 319 ----- js/src/cli/util/external-packages-plugin.ts | 43 - js/src/cli/util/external-packages.test.ts | 199 --- js/src/cli/util/pull.ts | 394 ------ js/src/cli/util/types.ts | 51 - js/src/durable-eval.ts | 1 - js/src/exports.ts | 20 - js/src/framework.test.ts | 30 - js/src/framework.ts | 87 +- js/src/framework2.ts | 96 +- js/src/gitutil.ts | 2 +- js/src/imports.test.ts | 95 +- js/src/node/index.ts | 4 +- js/src/parameters.test.ts | 7 - js/src/reporters/types.ts | 2 +- js/src/zod/zod-serialization-test-shared.ts | 70 - js/src/zod/zod-v3-serialization.test.ts | 234 ---- js/src/zod/zod-v4-serialization.test.ts | 141 -- .../api-compatibility.test.ts | 52 +- js/tsup.config.ts | 27 - knip.jsonc | 2 - package.json | 2 - pnpm-lock.yaml | 333 ++--- turbo.json | 32 - 55 files changed, 192 insertions(+), 6552 deletions(-) create mode 100644 .changeset/remove-js-braintrust-cli.md delete mode 100644 js/dev/authorize.ts delete mode 100644 js/dev/errorHandler.ts delete mode 100644 js/dev/index.ts delete mode 100644 js/dev/server.test.ts delete mode 100644 js/dev/server.ts delete mode 100644 js/dev/stream.ts delete mode 100644 js/dev/types.ts delete mode 100644 js/scripts/test-output.ts delete mode 100644 js/src/cli/auto-instrumentation.test.ts delete mode 100644 js/src/cli/debug-logging.test.ts delete mode 100644 js/src/cli/functions/infer-source.ts delete mode 100644 js/src/cli/functions/load-module.ts delete mode 100644 js/src/cli/functions/upload.test.ts delete mode 100644 js/src/cli/functions/upload.ts delete mode 100755 js/src/cli/index.ts delete mode 100644 js/src/cli/integration.test.ts delete mode 100644 js/src/cli/jest/nodeModulesPaths.ts delete mode 100644 js/src/cli/progress.ts delete mode 100644 js/src/cli/reporters/eval.ts delete mode 100644 js/src/cli/reporters/progress.ts delete mode 100644 js/src/cli/types.ts delete mode 100644 js/src/cli/util/bundle.ts delete mode 100644 js/src/cli/util/debug-logging.test.ts delete mode 100644 js/src/cli/util/debug-logging.ts delete mode 100644 js/src/cli/util/external-packages-plugin.test.ts delete mode 100644 js/src/cli/util/external-packages-plugin.ts delete mode 100644 js/src/cli/util/external-packages.test.ts delete mode 100644 js/src/cli/util/pull.ts delete mode 100644 js/src/cli/util/types.ts delete mode 100644 js/src/zod/zod-serialization-test-shared.ts delete mode 100644 js/src/zod/zod-v3-serialization.test.ts delete mode 100644 js/src/zod/zod-v4-serialization.test.ts diff --git a/.changeset/remove-js-braintrust-cli.md b/.changeset/remove-js-braintrust-cli.md new file mode 100644 index 000000000..9dfca85f2 --- /dev/null +++ b/.changeset/remove-js-braintrust-cli.md @@ -0,0 +1,5 @@ +--- +"braintrust": major +--- + +ref(v4)!: Remove JS Braintrust CLI diff --git a/.github/workflows/checks.yaml b/.github/workflows/checks.yaml index 74b97d78a..4b4d6ce8c 100644 --- a/.github/workflows/checks.yaml +++ b/.github/workflows/checks.yaml @@ -236,11 +236,10 @@ jobs: pnpm install --frozen-lockfile || { echo "Baseline install failed"; git checkout "$CURRENT_REF"; exit 0; } pnpm run build || { echo "Baseline build failed"; git checkout "$CURRENT_REF"; exit 0; } - mkdir -p "$BASELINE_DIR/js/dev" "$BASELINE_DIR/js/util" + mkdir -p "$BASELINE_DIR/js/util" cp -R js/dist "$BASELINE_DIR/js/" - cp -R js/dev/dist "$BASELINE_DIR/js/dev/" cp -R js/util/dist "$BASELINE_DIR/js/util/" - echo "Copied js/dist, js/dev/dist, and js/util/dist to baseline" + echo "Copied js/dist and js/util/dist to baseline" git checkout "$CURRENT_REF" @@ -473,17 +472,6 @@ jobs: - name: Run hermetic tests working-directory: ./js run: pnpm run test:checks - - name: Run zod v3 tests - if: matrix.zod-version == '3.25.34' - working-directory: ./js - run: | - pnpm test:zod-v3 - - name: Run zod v4 tests - if: matrix.zod-version == '4.2.1' - working-directory: ./js - run: | - pnpm test:zod-v4 - checks-passed: name: checks-passed needs: diff --git a/README.md b/README.md index d8fd417e4..d9858e7cb 100644 --- a/README.md +++ b/README.md @@ -37,7 +37,7 @@ Eval("Say Hi Bot", { Run it: ```bash -BRAINTRUST_API_KEY= npx braintrust eval tutorial.eval.ts +BRAINTRUST_API_KEY= npx bt eval tutorial.eval.ts ``` ## Packages diff --git a/integrations/otel-js/src/otel-compat.test.ts b/integrations/otel-js/src/otel-compat.test.ts index 6564da531..d06d9f843 100644 --- a/integrations/otel-js/src/otel-compat.test.ts +++ b/integrations/otel-js/src/otel-compat.test.ts @@ -12,7 +12,7 @@ import { getContextManager, BRAINTRUST_CURRENT_SPAN_STORE, _exportsForTestingOnly, - runEvaluator, + Eval, } from "braintrust"; import { BasicTracerProvider, @@ -371,7 +371,7 @@ describe("OTEL compatibility mode", () => { expect(trace2Id).not.toBe(trace3Id); }); - test("OTEL spans in experiment Eval() inherit experiment_id parent", async () => { + test("OTEL spans in Eval() inherit Braintrust parent context", async () => { const { tracer, exporter, processor } = setupOtelFixture( "experiment-eval-test", ); @@ -379,12 +379,11 @@ describe("OTEL compatibility mode", () => { // Capture BT span info from inside the task const btSpanInfo: Array<{ traceId: string; spanId: string }> = []; - // Use runEvaluator with null experiment to avoid API calls - const result = await runEvaluator( - null, + // Use the public Eval API without making Braintrust API calls. + const result = await Eval( + "test-eval-project", { - projectName: "test-eval-project", - evalName: "otel-eval-test", + experimentName: "otel-eval-test", data: [{ input: 1 }, { input: 2 }], task: async (input: number) => { // Capture the current BT span info @@ -408,9 +407,10 @@ describe("OTEL compatibility mode", () => { }, scores: [], }, - new NoopProgressReporter(), - [], - undefined, + { + noSendLogs: true, + progress: new NoopProgressReporter(), + }, ); // Verify we captured BT span info diff --git a/js/dev/authorize.ts b/js/dev/authorize.ts deleted file mode 100644 index 4f7592d17..000000000 --- a/js/dev/authorize.ts +++ /dev/null @@ -1,217 +0,0 @@ -import { Request, Response, NextFunction } from "express"; -import { IncomingHttpHeaders } from "http"; -import createError from "http-errors"; -import { LRUCache } from "../src/lru-cache"; -import { BraintrustState, LoginOptions, loginToState } from "../src/logger"; - -export interface RequestContext { - appOrigin: string; - token: string | undefined; - state: BraintrustState | undefined; - projectId: string | undefined; -} -declare module "express" { - interface Request { - ctx?: RequestContext; - } -} - -export function authorizeRequest( - req: Request, - res: Response, - next: NextFunction, -) { - try { - const ctx: RequestContext = { - appOrigin: extractAllowedOrigin(req.headers[ORIGIN_HEADER]), - token: undefined, - state: undefined, - projectId: parseHeader(req.headers, PROJECT_ID_HEADER), - }; - - // Extract token and data from request - if ( - req.headers.authorization || - req.headers[BRAINTRUST_AUTH_TOKEN_HEADER] - ) { - const tokenText = parseBraintrustAuthHeader(req.headers); - if (!tokenText) { - return next(createError(400, "Invalid authorization token format")); - } - ctx.token = tokenText.toLowerCase() === "null" ? undefined : tokenText; - } - - req.ctx = ctx; - - next(); // Proceed to next middleware/controller - } catch (e) { - next(e); - } -} - -const loginCache = new LRUCache({ - max: 32, // TODO: Make this configurable -}); - -async function cachedLogin(options: LoginOptions): Promise { - const key = JSON.stringify(options); - const cached = loginCache.get(key); - if (cached) { - return cached; - } - - const state = await loginToState(options); - loginCache.set(key, state); - return state; -} - -export function makeCheckAuthorized(allowedOrgName: string | undefined) { - return async (req: Request, _res: Response, next: NextFunction) => { - if (!req.ctx?.token) { - return next(createError(401, "Unauthorized")); - } - - try { - const orgName = parseHeader(req.headers, "x-bt-org-name"); - - if (!orgName) { - return next(createError(400, "Missing x-bt-org-name header")); - } - - if (allowedOrgName && allowedOrgName !== orgName) { - const errorMessage = `Org '${orgName}' is not allowed. Only org '${allowedOrgName}' is allowed.`; - return next(createError(403, errorMessage)); - } - - const state = await cachedLogin({ - apiKey: req.ctx?.token, - orgName: orgName, - }); - req.ctx.state = state; - next(); - } catch (e) { - console.error("Authorization error:", e); - return next(createError(401, "Unauthorized")); - } - }; -} - -function parseBraintrustAuthHeader( - headers: IncomingHttpHeaders | Record, -): string | undefined { - const tokenString = parseHeader(headers, BRAINTRUST_AUTH_TOKEN_HEADER); - return tokenString ?? parseAuthHeader(headers) ?? undefined; -} - -function parseHeader( - headers: IncomingHttpHeaders | Record, - headerName: string, -): string | undefined { - const token = headers[headerName]; - let tokenString; - if (typeof token === "string") { - tokenString = token; - } else if (Array.isArray(token) && token.length > 0) { - tokenString = token[0]; - } - - return tokenString; -} - -export type StaticOrigin = - | boolean - | string - | RegExp - | Array; - -export function checkOrigin( - requestOrigin: string | undefined, - callback: (err: Error | null, origin?: StaticOrigin) => void, -) { - if (!requestOrigin) { - return callback(null, true); - } - - // the origins can be glob patterns - for (const origin of WHITELISTED_ORIGINS || []) { - if ( - (origin instanceof RegExp && origin.test(requestOrigin)) || - origin === requestOrigin - ) { - return callback(null, requestOrigin); - } - } - - return callback(null, false); -} - -const BRAINTRUST_AUTH_TOKEN_HEADER = "x-bt-auth-token"; -const ORIGIN_HEADER = "origin"; -const PROJECT_ID_HEADER = "x-bt-project-id"; - -function extractAllowedOrigin(originHeader: string | undefined): string { - let allowedOrigin: string = MAIN_ORIGIN; - checkOrigin(originHeader, (err, origin) => { - if (!err && originHeader && origin) { - allowedOrigin = originHeader; - } - }); - return allowedOrigin; -} - -const MAIN_ORIGIN = "https://www.braintrust.dev"; -const WHITELISTED_ORIGINS = [ - MAIN_ORIGIN, - "https://www.braintrustdata.com", - new RegExp("^https://[^/]+\\.preview\\.braintrust\\.dev$"), -] - .concat( - process.env.WHITELISTED_ORIGIN ? [process.env.WHITELISTED_ORIGIN] : [], - ) - .concat( - process.env.BRAINTRUST_APP_URL ? [process.env.BRAINTRUST_APP_URL] : [], - ); - -function parseAuthHeader( - headers: Record, -) { - const authHeader = headers["authorization"]; - let authValue = null; - if (Array.isArray(authHeader)) { - authValue = authHeader[authHeader.length - 1]; - } else { - authValue = authHeader; - } - - if (!authValue) { - return null; - } - - const parts = authValue.split(" "); - if (parts.length !== 2) { - return null; - } - return parts[1]; -} - -export const baseAllowedHeaders = [ - "Content-Type", - "X-Amz-Date", - "Authorization", - "X-Api-Key", - "X-Amz-Security-Token", - "x-bt-auth-token", - "x-bt-parent", - // These are eval-specific - "x-bt-org-name", - "x-bt-project-id", - "x-bt-stream-fmt", - "x-bt-use-cache", - "x-stainless-os", - "x-stainless-lang", - "x-stainless-package-version", - "x-stainless-runtime", - "x-stainless-runtime-version", - "x-stainless-arch", - "x-bt-use-gateway", -]; diff --git a/js/dev/errorHandler.ts b/js/dev/errorHandler.ts deleted file mode 100644 index 12cd15e24..000000000 --- a/js/dev/errorHandler.ts +++ /dev/null @@ -1,37 +0,0 @@ -import { z } from "zod/v3"; -import { Request, Response, ErrorRequestHandler, NextFunction } from "express"; -import { HttpError } from "http-errors"; - -export const errorHandler: ErrorRequestHandler = ( - err: Error | HttpError, - req: Request, - res: Response, - next: NextFunction, -) => { - if ("status" in err) { - res.status(err.status).json({ - error: { - message: err.message, - status: err.status, - }, - }); - return; - } - if (err instanceof z.ZodError) { - res.status(400).json({ - error: { - message: "Invalid request", - errors: err.errors, - }, - }); - return; - } - - console.error("Internal server error", err); - res.status(500).json({ - error: { - message: "Internal server error", - status: 500, - }, - }); -}; diff --git a/js/dev/index.ts b/js/dev/index.ts deleted file mode 100644 index 7ac9af952..000000000 --- a/js/dev/index.ts +++ /dev/null @@ -1,5 +0,0 @@ -import { configureNode } from "../src/node/config"; - -configureNode(); - -export { runDevServer } from "./server"; diff --git a/js/dev/server.test.ts b/js/dev/server.test.ts deleted file mode 100644 index 2e1461a7e..000000000 --- a/js/dev/server.test.ts +++ /dev/null @@ -1,21 +0,0 @@ -import { describe, expect, test } from "vitest"; -import { BraintrustState } from "../src/logger"; -import { _exportsForTestingOnly } from "./server"; -import { evalBodySchema } from "./types"; - -describe("remote eval data", () => { - test("uses a named experiment as the data source", async () => { - const request = evalBodySchema.parse({ - name: "remote-evaluator", - data: { experiment_name: "source-experiment" }, - }); - const state = new BraintrustState({}); - - await expect( - _exportsForTestingOnly.getDataset(state, request.data), - ).resolves.toEqual({ - _type: "BaseExperiment", - name: "source-experiment", - }); - }); -}); diff --git a/js/dev/server.ts b/js/dev/server.ts deleted file mode 100644 index 5db8b467f..000000000 --- a/js/dev/server.ts +++ /dev/null @@ -1,407 +0,0 @@ -import express, { NextFunction, Request, Response } from "express"; -import cors from "cors"; -import { - BaseExperiment, - callEvaluatorData, - Eval, - EvalData, - EvalHooks, - EvalScorer, - EvaluatorDef, - OneOrMoreScores, - classifierName, - scorerName, -} from "../src/framework"; -import { errorHandler } from "./errorHandler"; -import { - authorizeRequest, - baseAllowedHeaders, - makeCheckAuthorized, - checkOrigin, -} from "./authorize"; -import { - type FunctionIdType as FunctionId, - type InvokeFunctionType as InvokeFunctionRequest, - type RunEvalType as RunEvalRequest, - type SSEProgressEventDataType as SSEProgressEventData, -} from "../src/generated_types"; -import { - BaseMetadata, - BraintrustState, - EvalCase, - getSpanParentObject, - initDataset, -} from "../src/logger"; -import { - BT_CURSOR_HEADER, - BT_FOUND_EXISTING_HEADER, - parseParent, -} from "../util/index"; -import { serializeSSEEvent } from "./stream"; -import { - evalBodySchema, - EvaluatorDefinitions, - EvaluatorManifest, - type SerializedParametersContainer, -} from "./types"; -import { EvalParameters, validateParameters } from "../src/eval-parameters"; -import { z } from "zod/v3"; -import { ValidationError } from "ajv"; -import { serializeRemoteEvalParametersContainer } from "../src/framework2"; - -export interface DevServerOpts { - host: string; - port: number; - orgName?: string; -} - -export function runDevServer( - // eslint-disable-next-line @typescript-eslint/no-explicit-any - evaluators: EvaluatorDef[], - opts: DevServerOpts, -) { - // eslint-disable-next-line @typescript-eslint/consistent-type-assertions - const allEvaluators: EvaluatorManifest = Object.fromEntries( - evaluators.map((evaluator) => [evaluator.evalName, evaluator]), - ) as EvaluatorManifest; - - globalThis._lazy_load = false; - - const app = express(); - - app.use(express.json({ limit: "1gb" })); - console.log("Starting server"); - app.use((req, res, next) => { - if (req.headers["access-control-request-private-network"]) { - res.setHeader("Access-Control-Allow-Private-Network", "true"); - } - next(); - }); - - const checkAuthorized = makeCheckAuthorized(opts.orgName); - - app.use( - cors({ - origin: checkOrigin, - methods: ["GET", "PATCH", "POST", "PUT", "DELETE", "OPTIONS"], - allowedHeaders: baseAllowedHeaders, - credentials: true, - exposedHeaders: [ - BT_CURSOR_HEADER, - BT_FOUND_EXISTING_HEADER, - "x-bt-span-id", - "x-bt-span-export", - ], - maxAge: 86400, - }), - ); - - app.use(authorizeRequest); - - app.get("/", (req, res) => { - res.send("Hello, world!"); - }); - - // List endpoint - returns all available evaluators and their metadata - app.get( - "/list", - checkAuthorized, - asyncHandler(async (req, res) => { - const evalDefs: EvaluatorDefinitions = {}; - - for (const [name, evaluator] of Object.entries(allEvaluators)) { - let parameters: SerializedParametersContainer | undefined; - - if (evaluator.parameters) { - const resolvedParams = await Promise.resolve(evaluator.parameters); - parameters = serializeRemoteEvalParametersContainer(resolvedParams); - } - - evalDefs[name] = { - parameters, - scores: (evaluator.scores ?? []).map((score, idx) => ({ - name: scorerName(score, idx), - })), - classifiers: (evaluator.classifiers ?? []).map((classifier, idx) => ({ - name: classifierName(classifier, idx), - })), - }; - } - - res.json(evalDefs); - }), - ); - - app.post( - "/eval", - checkAuthorized, - asyncHandler(async (req, res) => { - const { - name, - parameters, - parent, - experiment_name, - project_id, - data, - scores, - stream, - } = evalBodySchema.parse(req.body); - - if (!req.ctx?.state) { - res - .status(500) - .json({ error: "Braintrust state not initialized in request" }); - return; - } - const state = req.ctx.state; - - const evaluator = allEvaluators[name]; - if (!evaluator) { - res.status(404).json({ error: `Evaluator '${name}' not found` }); - return; - } - - if (evaluator.parameters) { - try { - await validateParameters(parameters ?? {}, evaluator.parameters); - } catch (e) { - console.error("Error validating parameters", e); - if ( - e instanceof z.ZodError || - e instanceof ValidationError || - e instanceof Error - ) { - res.status(400).json({ - error: e.message, - }); - return; - } - throw e; - } - } - - const resolvedData = await getDataset(state, data); - const evalData = callEvaluatorData(resolvedData); - console.log("Starting eval", evaluator.evalName); - - // Set up SSE headers - if (stream) { - res.setHeader("Content-Type", "text/event-stream"); - res.setHeader("Cache-Control", "no-cache"); - res.setHeader("Connection", "keep-alive"); - } else { - res.setHeader("Content-Type", "application/json"); - } - - const task = async ( - input: unknown, - hooks: EvalHooks, - ) => { - const result = await evaluator.task(input, hooks); - - hooks.reportProgress({ - format: "code", - output_type: "completion", - event: "json_delta", - data: JSON.stringify(result), - }); - return result; - }; - - try { - const summary = await Eval( - "worker-thread", - { - ...evaluator, - data: evalData.data, - scores: (evaluator.scores ?? []).concat( - scores?.map((score) => - makeScorer( - state, - score.name, - score.function_id, - req.ctx?.projectId, - ), - ) ?? [], - ), - task, - state, - experimentName: experiment_name ?? undefined, - projectId: project_id ?? undefined, - }, - { - // Avoid printing the bar to the console. - progress: { - start: () => {}, - stop: () => { - console.log("Finished running experiment"); - }, - increment: () => {}, - }, - stream: (data: SSEProgressEventData) => { - if (stream) { - res.write( - serializeSSEEvent({ - event: "progress", - data: JSON.stringify(data), - }), - ); - } - }, - onStart: (metadata) => { - if (stream) { - res.write( - serializeSSEEvent({ - event: "start", - data: JSON.stringify(metadata), - }), - ); - } - }, - parent: parseParent(parent), - parameters: parameters ?? {}, - }, - ); - - if (stream) { - res.write( - serializeSSEEvent({ - event: "summary", - data: JSON.stringify(summary.summary), - }), - ); - res.write( - serializeSSEEvent({ - event: "done", - data: "", - }), - ); - } else { - res.json(summary.summary); - } - } catch (e) { - console.error("Error running eval", e); - if (stream) { - res.write( - serializeSSEEvent({ - event: "error", - data: JSON.stringify(e), - }), - ); - } else { - res.status(500).json({ error: e }); - } - } finally { - res.end(); - } - }), - ); - - app.use(errorHandler); - - // Start the server - app.listen(opts.port, opts.host, () => { - console.log(`Dev server running at http://${opts.host}:${opts.port}`); - }); -} -const asyncHandler = - (fn: (req: Request, res: Response, next: NextFunction) => Promise) => - (req: Request, res: Response, next: NextFunction) => { - Promise.resolve(fn(req, res, next)).catch(next); - }; - -async function getDataset( - state: BraintrustState, - data: RunEvalRequest["data"], -): Promise> { - if ("project_name" in data) { - return initDataset({ - state, - project: data.project_name, - dataset: data.dataset_name, - version: data.dataset_version ?? undefined, - environment: data.dataset_environment ?? undefined, - _internal_btql: data._internal_btql ?? undefined, - }); - } else if ("dataset_id" in data) { - const datasetInfo = await getDatasetById({ - state, - datasetId: data.dataset_id, - }); - return initDataset({ - state, - projectId: datasetInfo.projectId, - dataset: datasetInfo.dataset, - version: data.dataset_version ?? undefined, - environment: data.dataset_environment ?? undefined, - _internal_btql: data._internal_btql ?? undefined, - }); - } else if ("experiment_name" in data) { - return BaseExperiment({ name: data.experiment_name }); - } else { - // eslint-disable-next-line @typescript-eslint/consistent-type-assertions - return data.data as EvalCase[]; - } -} - -const datasetFetchSchema = z.object({ - project_id: z.string(), - name: z.string(), -}); -async function getDatasetById({ - state, - datasetId, -}: { - state: BraintrustState; - datasetId: string; -}): Promise<{ projectId: string; dataset: string }> { - const dataset = await state.appConn().post_json("api/dataset/get", { - id: datasetId, - }); - const parsed = z.array(datasetFetchSchema).parse(dataset); - if (parsed.length === 0) { - throw new Error(`Dataset '${datasetId}' not found`); - } - return { projectId: parsed[0].project_id, dataset: parsed[0].name }; -} - -function makeScorer( - state: BraintrustState, - name: string, - score: FunctionId, - projectId: string | undefined, -): EvalScorer { - const ret = async (input: EvalCase) => { - const request: InvokeFunctionRequest = { - ...score, - input, - parent: await getSpanParentObject().export(), - stream: false, - mode: "auto", - strict: true, - }; - const headers: Record = { - Accept: "application/json", - }; - if (projectId) { - headers["x-bt-project-id"] = projectId; - } - const result = await state.proxyConn().post(`function/invoke`, request, { - headers, - }); - const data = await result.json(); - // NOTE: Ideally we can parse this value with a zod schema. - // eslint-disable-next-line @typescript-eslint/consistent-type-assertions - return data as OneOrMoreScores; - }; - - Object.defineProperties(ret, { - name: { value: `Remote eval scorer (${name})` }, - }); - - return ret; -} - -export const _exportsForTestingOnly = { - getDataset, -}; diff --git a/js/dev/stream.ts b/js/dev/stream.ts deleted file mode 100644 index 7bca3612b..000000000 --- a/js/dev/stream.ts +++ /dev/null @@ -1,14 +0,0 @@ -export interface RawSSEEvent { - id?: string; - event?: string; - data: string; -} - -export function serializeSSEEvent(event: RawSSEEvent): string { - return ( - Object.entries(event) - .filter(([_key, value]) => value !== undefined) - .map(([key, value]) => `${key}: ${value}`) - .join("\n") + "\n\n" - ); -} diff --git a/js/dev/types.ts b/js/dev/types.ts deleted file mode 100644 index da992efea..000000000 --- a/js/dev/types.ts +++ /dev/null @@ -1,123 +0,0 @@ -import { - FunctionId as functionIdSchema, - InvokeParent as invokeParentSchema, - RunEval as runEvalSchema, - PromptData as promptDataSchema, -} from "../src/generated_types"; -import { z } from "zod/v3"; -import { EvaluatorDef } from "../src/framework"; -import { BaseMetadata } from "../src/logger"; - -export const evalBodySchema = z.object({ - name: z.string(), - parameters: z.record(z.string(), z.unknown()).nullish(), - data: runEvalSchema.shape.data, - scores: z - .array( - z.object({ - function_id: functionIdSchema, - name: z.string(), - }), - ) - .nullish(), - experiment_name: z.string().nullish(), - project_id: z.string().nullish(), - parent: invokeParentSchema.optional(), - stream: z.boolean().optional(), -}); - -export type EvaluatorManifest = Record< - string, - EvaluatorDef ->; - -export const staticParametersSchema = z.record( - z.string(), - z.union([ - z.object({ - type: z.literal("prompt"), - default: promptDataSchema.optional(), - description: z.string().optional(), - }), - z.object({ - type: z.literal("model"), - default: z.string().optional(), - description: z.string().optional(), - }), - z.object({ - type: z.literal("data"), - schema: z.record(z.unknown()), - default: z.unknown().optional(), - description: z.string().optional(), - }), - ]), -); - -export type StaticParametersSchema = z.infer; - -const evalParametersSerializedSchema = staticParametersSchema; -export type EvalParameterSerializedSchema = z.infer< - typeof evalParametersSerializedSchema ->; - -export const parametersSchema = z.object({ - type: z.literal("object"), - properties: z.record(z.string(), z.record(z.unknown())), - required: z.array(z.string()).optional(), - additionalProperties: z.boolean().optional(), -}); - -export type ParametersSchema = z.infer; - -const parametersSourceSchema = z.object({ - parametersId: z.string().optional(), - slug: z.string(), - name: z.string(), - projectId: z.string().optional(), - version: z.string().optional(), -}); - -export type ParametersSource = z.infer; - -const parametersContainerSchema = z.object({ - type: z.literal("braintrust.parameters"), - schema: parametersSchema, - source: parametersSourceSchema, -}); - -export type ParametersContainer = z.infer; - -const staticParametersContainerSchema = z.object({ - type: z.literal("braintrust.staticParameters"), - schema: staticParametersSchema, - source: z.null().nullish(), -}); - -export type StaticParametersContainer = z.infer< - typeof staticParametersContainerSchema ->; - -export const serializedParametersContainerSchema = z.union([ - parametersContainerSchema, - staticParametersContainerSchema, - // keeping this type here since old versions of the SDK will still pass the unwrapped schema and we need to handle this in the app - staticParametersSchema, -]); - -export type SerializedParametersContainer = z.infer< - typeof serializedParametersContainerSchema ->; - -export const evaluatorDefinitionSchema = z.object({ - parameters: serializedParametersContainerSchema.nullish(), - scores: z.array(z.object({ name: z.string() })).optional(), - classifiers: z.array(z.object({ name: z.string() })).optional(), -}); -export type EvaluatorDefinition = z.infer; - -export const evaluatorDefinitionsSchema = z.record( - z.string(), - evaluatorDefinitionSchema, -); - -export type EvaluatorDefinitions = z.infer; diff --git a/js/eslint.config.ts b/js/eslint.config.ts index 04e2b8f78..80829409c 100644 --- a/js/eslint.config.ts +++ b/js/eslint.config.ts @@ -12,21 +12,18 @@ const tsupConfig = Array.isArray(tsupConfigImport) ? tsupConfigImport : tsupConfigImport.default || []; -// Extract entry points from tsup config, excluding CLI -const entryFiles = tsupConfig - .flatMap((config) => { - const entries = Array.isArray(config.entry) - ? config.entry - : Object.values(config.entry); - return entries; - }) - .filter((entry) => !entry.includes("cli")); +// Extract entry points from tsup config +const entryFiles = tsupConfig.flatMap((config) => { + const entries = Array.isArray(config.entry) + ? config.entry + : Object.values(config.entry); + return entries; +}); export default [ { ignores: [ "dist/**", - "dev/dist/**", "util/dist/**", "node_modules/**", "vendor/**", @@ -164,38 +161,6 @@ export default [ "no-restricted-properties": "off", }, }, - { - files: ["src/**/*.ts", "src/**/*.tsx"], - ignores: [ - "src/cli/**", - "src/debug-logger.ts", - "src/framework.ts", - "src/framework2.ts", - "src/isomorph.ts", - "src/sandbox.ts", - "src/template/**", - "src/reporters/**", - "src/prompt-cache/**", - "src/eval-parameters.ts", - "src/wrappers/**", - "src/instrumentation/**", - "src/auto-instrumentations/**", - ], - rules: { - "no-restricted-imports": [ - "error", - { - patterns: [ - { - group: ["**/cli", "**/cli/**", "./cli", "./cli/**"], - message: - "Importing from 'cli' directory is not allowed. CLI code should not be imported by SDK code.", - }, - ], - }, - ], - }, - }, { files: ["src/**/*.ts", "src/**/*.tsx"], ignores: [...entryFiles, "**/*.test.ts", "**/*.test.tsx"], diff --git a/js/package.json b/js/package.json index 3841c8b24..2f9375376 100644 --- a/js/package.json +++ b/js/package.json @@ -19,7 +19,6 @@ "./dist/index.d.mts": "./dist/browser.d.mts" }, "bin": { - "braintrust": "./dist/cli.js", "bt": "./bin/bt" }, "exports": { @@ -77,12 +76,6 @@ "module": "./dist/index.mjs", "require": "./dist/index.js" }, - "./dev": { - "types": "./dev/dist/index.d.ts", - "import": "./dev/dist/index.mjs", - "module": "./dev/dist/index.mjs", - "require": "./dev/dist/index.js" - }, "./util": { "types": "./util/dist/index.d.ts", "import": "./util/dist/index.mjs", @@ -133,7 +126,6 @@ }, "files": [ "dist/**/*", - "dev/dist/**/*", "util/dist/**/*", "bin/bt", "scripts/bt-helper.js", @@ -146,9 +138,9 @@ "build": "cross-env NODE_OPTIONS=\"--max-old-space-size=8192\" tsup", "check:typings": "tsc --noEmit && tsc --noEmit -p tsconfig.test.json", "watch": "tsup --watch", - "clean": "rm -r dist/* && rm -r dev/dist/* && rm -r util/dist/*", + "clean": "rm -r dist/* && rm -r util/dist/*", "docs": "typedoc --options typedoc.json src/node/index.ts", - "test": "vitest run --exclude \"src/wrappers/**/*.test.ts\" --exclude \"src/otel/**/*.test.ts\" --exclude \"smoke/**/*.test.ts\" --exclude \"src/zod/**/*.test.ts\" --exclude \"tests/api-compatibility/**\"", + "test": "vitest run --exclude \"src/wrappers/**/*.test.ts\" --exclude \"src/otel/**/*.test.ts\" --exclude \"smoke/**/*.test.ts\" --exclude \"tests/api-compatibility/**\"", "test:core": "pnpm prune && pnpm test", "test:checks": "pnpm run test:core && pnpm run test:vitest", "test:all": "pnpm run test:checks", @@ -158,18 +150,13 @@ "test:ai-sdk-v1": "vitest run src/wrappers/ai-sdk-v1.test.ts", "test:ai-sdk-v2": "vitest run src/wrappers/ai-sdk-v2.test.ts src/wrappers/ai-sdk-v1.test.ts", "test:ai-sdk-v3": "vitest run src/wrappers/ai-sdk-v3.test.ts", - "test:zod-v3": "vitest run src/zod/zod-v3-serialization.test.ts", - "test:zod-v4": "vitest run src/zod/zod-v4-serialization.test.ts", "test:mastra": "vitest run src/wrappers/mastra.test.ts", "test:vitest": "pnpm --filter @braintrust/vitest-wrapper-tests test", - "test:output": "tsx scripts/test-output.ts --with-comparison --with-metrics --with-progress", "bench": "tsx src/queue.bench.ts", "lint": "eslint .", "fix:lint": "eslint --fix .", "playground": "tsx playground.ts", "playground:auto": "mkdir -p .context && pnpm exec esbuild playground.ts --platform=node --format=esm --outfile=.context/playground.auto.mjs && node --import ./dist/auto-instrumentations/hook.mjs ./.context/playground.auto.mjs", - "playground:cli:push": "node dist/cli.js push playground.ts", - "playground:cli:eval": "node dist/cli.js eval playground.ts", "yalc:publish": "yalc publish" }, "author": "", @@ -179,17 +166,10 @@ }, "devDependencies": { "@anthropic-ai/sdk": "^0.60.0", - "@nodelib/fs.walk": "^1.2.8", - "@types/argparse": "^2.0.14", "@types/async": "^3.2.24", - "@types/cli-progress": "^3.11.5", - "@types/cors": "^2.8.17", "@types/esquery": "^1.5.4", - "@types/express": "^5.0.0", - "@types/http-errors": "^2.0.4", "@types/mustache": "^4.2.5", "@types/node": "^20.10.5", - "@types/pluralize": "^0.0.30", "@typescript-eslint/eslint-plugin": "^8.49.0", "@typescript-eslint/parser": "^8.49.0", "ai": "^6.0.0", @@ -216,31 +196,21 @@ "zod": "^3.25.34" }, "dependencies": { - "@next/env": "^14.2.3", "@vercel/functions": "^1.0.2", "acorn": "^8.16.0", "acorn-import-attributes": "^1.9.5", "ajv": "^8.20.0", - "argparse": "^2.0.1", "astring": "^1.9.0", "cjs-module-lexer": "^2.2.0", - "cli-progress": "^3.12.0", - "cli-table3": "^0.6.5", - "cors": "^2.8.5", "dotenv": "^16.4.5", "esbuild": "0.28.1", "esquery": "^1.7.0", "eventsource-parser": "^1.1.2", - "express": "^5.2.1", - "http-errors": "^2.0.0", "meriyah": "^6.1.4", - "minimatch": "^10.2.5", "module-details-from-path": "^1.0.4", "mustache": "^4.2.0", - "pluralize": "^8.0.0", "semifies": "^1.0.0", "source-map": "^0.7.4", - "termi-link": "^1.0.1", "unplugin": "^2.3.5", "uuid": "^11.1.1", "zod-to-json-schema": "^3.25.0" diff --git a/js/scripts/test-output.ts b/js/scripts/test-output.ts deleted file mode 100644 index 8a9ca0858..000000000 --- a/js/scripts/test-output.ts +++ /dev/null @@ -1,194 +0,0 @@ -#!/usr/bin/env tsx - -/** - * Test script to preview CLI output formatting without running actual experiments. - * - * Usage: - * pnpm tsx scripts/test-output.ts - * pnpm tsx scripts/test-output.ts --with-comparison - * pnpm tsx scripts/test-output.ts --with-metrics - * pnpm tsx scripts/test-output.ts --with-progress - */ - -import { formatExperimentSummary } from "../src/framework"; -import type { ExperimentSummary } from "../src/logger"; -import { BarProgressReporter } from "../src/cli/progress"; - -function createSampleSummary(options: { - withComparison?: boolean; - withMetrics?: boolean; -}): ExperimentSummary { - const summary: ExperimentSummary = { - projectName: "my-project", - experimentName: "test-experiment", - experimentId: "exp-123", - projectId: "proj-456", - experimentUrl: - "https://www.braintrust.dev/app/my-org/p/my-project/experiments/test-experiment", - projectUrl: "https://www.braintrust.dev/app/my-org/p/my-project", - scores: { - accuracy: { - name: "Accuracy", - score: 0.95, - diff: options.withComparison ? 0.02 : undefined, - improvements: options.withComparison ? 15 : 0, - regressions: options.withComparison ? 3 : 0, - }, - factuality: { - name: "Factuality", - score: 0.87, - diff: options.withComparison ? -0.01 : undefined, - improvements: options.withComparison ? 8 : 0, - regressions: options.withComparison ? 12 : 0, - }, - }, - }; - - if (options.withMetrics) { - summary.metrics = { - duration: { - name: "Duration", - metric: 1.23, - unit: "s", - diff: options.withComparison ? -0.15 : undefined, - improvements: options.withComparison ? 10 : 0, - regressions: options.withComparison ? 5 : 0, - }, - llm_duration: { - name: "LLM duration", - metric: 0.45, - unit: "s", - diff: options.withComparison ? -0.08 : undefined, - improvements: options.withComparison ? 12 : 0, - regressions: options.withComparison ? 3 : 0, - }, - prompt_tokens: { - name: "Prompt tokens", - metric: 4282, - unit: "", - diff: options.withComparison ? 0.035 : undefined, - improvements: options.withComparison ? 0 : 0, - regressions: options.withComparison ? 0 : 0, - }, - completion_tokens: { - name: "Completion tokens", - metric: 310, - unit: "", - diff: options.withComparison ? -0.08 : undefined, - improvements: options.withComparison ? 0 : 0, - regressions: options.withComparison ? 0 : 0, - }, - total_tokens: { - name: "Total tokens", - metric: 4592, - unit: "", - diff: options.withComparison ? 0.027 : undefined, - improvements: options.withComparison ? 0 : 0, - regressions: options.withComparison ? 0 : 0, - }, - estimated_cost: { - name: "Estimated cost", - metric: 0.01, - unit: "$", - diff: options.withComparison ? 0.2 : undefined, - improvements: options.withComparison ? 0 : 0, - regressions: options.withComparison ? 0 : 0, - }, - }; - } - - if (options.withComparison) { - summary.comparisonExperimentName = "baseline-experiment"; - } - - return summary; -} - -async function simulateProgress(evaluatorName: string, total: number) { - const progressReporter = new BarProgressReporter(); - progressReporter.start(evaluatorName, total); - - // Simulate progress with random delays - for (let i = 0; i < total; i++) { - await new Promise((resolve) => - setTimeout(resolve, 50 + Math.random() * 100), - ); - progressReporter.increment(evaluatorName); - } - - progressReporter.stop(); -} - -async function main() { - const args = process.argv.slice(2); - const withComparison = args.includes("--with-comparison"); - const withMetrics = args.includes("--with-metrics"); - const withProgress = args.includes("--with-progress"); - - console.log("\n" + "=".repeat(60)); - console.log("Testing CLI Output Formatting"); - console.log("=".repeat(60) + "\n"); - - // Test 1: Basic summary - console.log("1. Basic Summary (scores only):"); - console.log("-".repeat(60)); - - if (withProgress) { - await simulateProgress("test-experiment", 10); - } - - const basicSummary = createSampleSummary({}); - console.log(formatExperimentSummary(basicSummary)); - console.log("\n"); - - // Test 2: With comparison - if (withComparison) { - console.log("2. Summary with Comparison:"); - console.log("-".repeat(60)); - - if (withProgress) { - await simulateProgress("test-experiment", 15); - } - - const comparisonSummary = createSampleSummary({ withComparison: true }); - console.log(formatExperimentSummary(comparisonSummary)); - console.log("\n"); - } - - // Test 3: With metrics - if (withMetrics) { - console.log("3. Summary with Metrics:"); - console.log("-".repeat(60)); - - if (withProgress) { - await simulateProgress("test-experiment", 20); - } - - const metricsSummary = createSampleSummary({ withMetrics: true }); - console.log(formatExperimentSummary(metricsSummary)); - console.log("\n"); - } - - // Test 4: Full summary - console.log("4. Full Summary (scores + metrics + comparison):"); - console.log("-".repeat(60)); - - if (withProgress) { - await simulateProgress("test-experiment", 25); - } - - const fullSummary = createSampleSummary({ - withComparison: true, - withMetrics: true, - }); - console.log(formatExperimentSummary(fullSummary)); - console.log("\n"); - - console.log("=".repeat(60)); - console.log( - "Done! Use --with-comparison, --with-metrics, and --with-progress flags to see more variations.", - ); - console.log("=".repeat(60) + "\n"); -} - -main(); diff --git a/js/smoke/shared/src/suites/import-verification.ts b/js/smoke/shared/src/suites/import-verification.ts index 492ba2f0e..cf422006b 100644 --- a/js/smoke/shared/src/suites/import-verification.ts +++ b/js/smoke/shared/src/suites/import-verification.ts @@ -54,9 +54,6 @@ export interface BraintrustModule { Eval?: unknown; EvalResultWithSummary?: unknown; Reporter?: unknown; - runEvaluator?: unknown; - buildLocalSummary?: unknown; - reportFailures?: unknown; defaultErrorScoreHandler?: unknown; // Tracing (REQUIRED) @@ -221,27 +218,6 @@ export const testEvalExports = register("testEvalExports", async (module) => { assertDefined(module.Reporter, "Reporter must exist"); assertType(module.Reporter, "function", "Reporter must be a function"); - assertDefined(module.runEvaluator, "runEvaluator must exist"); - assertType( - module.runEvaluator, - "function", - "runEvaluator must be a function", - ); - - assertDefined(module.buildLocalSummary, "buildLocalSummary must exist"); - assertType( - module.buildLocalSummary, - "function", - "buildLocalSummary must be a function", - ); - - assertDefined(module.reportFailures, "reportFailures must exist"); - assertType( - module.reportFailures, - "function", - "reportFailures must be a function", - ); - assertDefined( module.defaultErrorScoreHandler, "defaultErrorScoreHandler must exist", @@ -252,7 +228,7 @@ export const testEvalExports = register("testEvalExports", async (module) => { "defaultErrorScoreHandler must be a function", ); - return "Eval exports verified (7 runtime exports)"; + return "Eval exports verified (4 runtime exports)"; }); export const testTracingExports = register( diff --git a/js/src/cli/auto-instrumentation.test.ts b/js/src/cli/auto-instrumentation.test.ts deleted file mode 100644 index 0b9f41495..000000000 --- a/js/src/cli/auto-instrumentation.test.ts +++ /dev/null @@ -1,306 +0,0 @@ -import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -import * as fs from "node:fs/promises"; -import * as os from "node:os"; -import * as path from "node:path"; -import { execFile } from "node:child_process"; -import { promisify } from "node:util"; -import { fileURLToPath } from "node:url"; -import { newGlobalTracingChannel } from "../global-instrumentation-hooks"; -import { initializeHandles } from "./index"; -import type { FileHandle } from "./types"; - -vi.mock("./functions/load-module", () => ({ - loadModule: () => ({ - evaluators: {}, - functions: [], - parameters: [], - prompts: [], - reporters: {}, - }), -})); - -const googleGenAIChannel = "orchestrion:@google/genai:models.generateContent"; -const anthropicChannel = "orchestrion:@anthropic-ai/sdk:messages.create"; -const execFileAsync = promisify(execFile); -const loadModulePath = fileURLToPath( - new URL("./functions/load-module.ts", import.meta.url), -); -const globalHooksPath = fileURLToPath( - new URL("../global-instrumentation-hooks.ts", import.meta.url), -); -const debugLoggerPath = fileURLToPath( - new URL("../debug-logger.ts", import.meta.url), -); - -describe("eval auto-instrumentation", () => { - let fixtureDir: string; - let handles: Record = {}; - - beforeEach(async () => { - fixtureDir = await fs.mkdtemp( - path.join(os.tmpdir(), "braintrust-eval-instrumentation-"), - ); - await writeFixturePackages(fixtureDir); - }); - - afterEach(async () => { - vi.unstubAllEnvs(); - await Promise.all(Object.values(handles).map((handle) => handle.destroy())); - await Promise.all( - [ - ...new Set( - Object.values(handles) - .map((handle) => handle.bundleFile) - .filter( - (bundleFile): bundleFile is string => bundleFile !== undefined, - ) - .map((bundleFile) => path.dirname(bundleFile)), - ), - ].map((directory) => fs.rm(directory, { recursive: true, force: true })), - ); - handles = {}; - await fs.rm(fixtureDir, { recursive: true, force: true }); - }); - - it.each([ - [ - "directly", - `export { Models } from "@google/genai";`, - (exports: Record) => { - const Models = exports.Models as new () => { - generateContentInternal(input: string): Promise<{ text: string }>; - }; - return new Models().generateContentInternal("payload"); - }, - ], - [ - "through a dependency", - `export { invoke } from "google-genai-client";`, - (exports: Record) => - (exports.invoke as (input: string) => Promise<{ text: string }>)( - "payload", - ), - ], - ])( - "instruments @google/genai when imported %s", - async (_label, source, invoke) => { - const output = await buildEval(fixtureDir, source); - - expect(output).toContain(googleGenAIChannel); - - const lifecycle: string[] = []; - const hook = newGlobalTracingChannel(googleGenAIChannel); - const handlers = { - asyncEnd: () => lifecycle.push("asyncEnd"), - asyncStart: () => lifecycle.push("asyncStart"), - end: () => lifecycle.push("end"), - start: () => lifecycle.push("start"), - }; - hook.subscribe(handlers); - - try { - const loadedModule = { exports: {} as Record }; - Function( - "module", - "exports", - output, - )(loadedModule, loadedModule.exports); - - await expect(invoke(loadedModule.exports)).resolves.toEqual({ - text: "payload", - }); - expect(lifecycle).toEqual(["start", "end", "asyncStart", "asyncEnd"]); - } finally { - hook.unsubscribe(handlers); - } - }, - ); - - it("respects BRAINTRUST_DISABLE_INSTRUMENTATION", async () => { - vi.stubEnv("BRAINTRUST_DISABLE_INSTRUMENTATION", "google-genai"); - - const output = await buildEval( - fixtureDir, - `export { Models } from "@google/genai";`, - ); - - expect(output).not.toContain(googleGenAIChannel); - }); - - it("instruments the final uploaded bundle", async () => { - await buildEval(fixtureDir, `export { Models } from "@google/genai";`); - - await handles[path.join(fixtureDir, "instrumentation.eval.ts")].bundle(); - const output = await fs.readFile( - handles[path.join(fixtureDir, "instrumentation.eval.ts")].bundleFile!, - "utf8", - ); - - expect(output).toContain(googleGenAIChannel); - }); - - it.each([ - ["instruments", undefined, ["start", "end", "asyncStart", "asyncEnd"]], - ["respects opt-out for", "anthropic", []], - ])("%s external eval dependencies", async (_label, disabled, expected) => { - const runnerPath = path.join(fixtureDir, "external-eval-runner.cjs"); - const evalFile = path.join(fixtureDir, "external.eval.ts"); - await fs.writeFile(evalFile, ""); - await fs.writeFile( - runnerPath, - `require("tsx/cjs"); -require("node:module").register = () => {}; -const { loadModule } = require(${JSON.stringify(loadModulePath)}); -const { newGlobalTracingChannel } = require(${JSON.stringify(globalHooksPath)}); -const lifecycle = []; -const channel = newGlobalTracingChannel(${JSON.stringify(anthropicChannel)}); -const handlers = Object.fromEntries( - ["start", "end", "asyncStart", "asyncEnd"].map((name) => [name, () => lifecycle.push(name)]), -); -channel.subscribe(handlers); -loadModule({ - inFile: ${JSON.stringify(evalFile)}, - moduleText: 'const { Messages } = require("@anthropic-ai/sdk/resources/messages/messages.js"); globalThis.__externalEvalResult = new Messages().create("payload");', -}); -Promise.resolve(globalThis.__externalEvalResult).then((result) => { - channel.unsubscribe(handlers); - process.stdout.write(JSON.stringify({ lifecycle, result })); -});`, - ); - - const env = { ...process.env }; - if (disabled) { - env.BRAINTRUST_DISABLE_INSTRUMENTATION = disabled; - } else { - delete env.BRAINTRUST_DISABLE_INSTRUMENTATION; - } - const { stdout } = await execFileAsync(process.execPath, [runnerPath], { - env, - }); - - expect(JSON.parse(stdout)).toEqual({ - lifecycle: expected, - result: { text: "payload" }, - }); - }); - - it("reports unexpected runtime instrumentation failures", async () => { - const runnerPath = path.join( - fixtureDir, - "failed-instrumentation-runner.cjs", - ); - const evalFile = path.join(fixtureDir, "failed-instrumentation.eval.ts"); - await fs.writeFile(evalFile, ""); - await fs.writeFile( - runnerPath, - `require("tsx/cjs"); -require("node:module").register = () => { throw new Error("setup failed"); }; -require(${JSON.stringify(debugLoggerPath)}).setGlobalDebugLogLevel("warn"); -const { loadModule } = require(${JSON.stringify(loadModulePath)}); -loadModule({ inFile: ${JSON.stringify(evalFile)}, moduleText: "" }); -process.stdout.write("loaded");`, - ); - - const { stderr, stdout } = await execFileAsync( - process.execPath, - [runnerPath], - { - env: { ...process.env, BRAINTRUST_DEBUG_LOG_LEVEL: "warn" }, - }, - ); - - expect(stdout).toBe("loaded"); - expect(stderr).toContain( - "Failed to enable auto-instrumentation for external eval dependencies", - ); - expect(stderr).toContain("setup failed"); - }); - - async function buildEval(fixtureDir: string, source: string) { - const evalFile = path.join(fixtureDir, "instrumentation.eval.ts"); - await fs.writeFile(evalFile, source); - handles = await initializeHandles({ files: [evalFile], mode: "eval" }); - - const result = await handles[evalFile].rebuild(); - if (result.type !== "success") { - throw result.error; - } - - return result.result.outputFiles?.[0].text ?? ""; - } -}); - -async function writeFixturePackages(fixtureDir: string) { - const googlePackageDir = path.join(fixtureDir, "node_modules/@google/genai"); - const indirectPackageDir = path.join( - fixtureDir, - "node_modules/google-genai-client", - ); - const anthropicPackageDir = path.join( - fixtureDir, - "node_modules/@anthropic-ai/sdk", - ); - await fs.mkdir(path.join(googlePackageDir, "dist/node"), { - recursive: true, - }); - await fs.mkdir(indirectPackageDir, { recursive: true }); - await fs.mkdir(path.join(anthropicPackageDir, "resources/messages"), { - recursive: true, - }); - - await Promise.all([ - fs.writeFile( - path.join(googlePackageDir, "package.json"), - JSON.stringify({ - name: "@google/genai", - version: "1.50.0", - type: "module", - exports: "./dist/node/index.mjs", - }), - ), - fs.writeFile( - path.join(googlePackageDir, "dist/node/index.mjs"), - `export class Models { - async generateContentInternal(input) { - return { text: input }; - } - }`, - ), - fs.writeFile( - path.join(indirectPackageDir, "package.json"), - JSON.stringify({ - name: "google-genai-client", - version: "1.0.0", - type: "module", - exports: "./index.mjs", - }), - ), - fs.writeFile( - path.join(indirectPackageDir, "index.mjs"), - `import { Models } from "@google/genai"; - export function invoke(input) { - return new Models().generateContentInternal(input); - }`, - ), - fs.writeFile( - path.join(anthropicPackageDir, "package.json"), - JSON.stringify({ - name: "@anthropic-ai/sdk", - version: "0.60.0", - exports: { - "./resources/messages/messages.js": - "./resources/messages/messages.js", - }, - }), - ), - fs.writeFile( - path.join(anthropicPackageDir, "resources/messages/messages.js"), - `class Messages { - async create(input) { - return { text: input }; - } - } - module.exports = { Messages };`, - ), - ]); -} diff --git a/js/src/cli/debug-logging.test.ts b/js/src/cli/debug-logging.test.ts deleted file mode 100644 index 49e352679..000000000 --- a/js/src/cli/debug-logging.test.ts +++ /dev/null @@ -1,71 +0,0 @@ -import { beforeEach, describe, expect, test, vi } from "vitest"; - -vi.mock("@next/env", () => ({ - loadEnvConfig: vi.fn(), -})); - -vi.mock("dotenv", () => ({ - config: vi.fn(() => ({})), -})); - -vi.mock("./index", () => ({ - handleBuildFailure: vi.fn(), - initializeHandles: vi.fn(), -})); - -vi.mock("../logger", async (importOriginal) => { - const actual = await importOriginal(); - return { - ...actual, - login: vi.fn().mockResolvedValue(undefined), - }; -}); - -import { login } from "../logger"; -import { resetDebugLoggingArgsForTests } from "./util/debug-logging"; -import { loadCLIEnv } from "./util/bundle"; - -describe("CLI debug logging", () => { - beforeEach(() => { - vi.clearAllMocks(); - resetDebugLoggingArgsForTests(); - }); - - test("loadCLIEnv forwards debugLogLevel to login", async () => { - await loadCLIEnv({ - verbose: false, - api_key: "test-key", - org_name: "test-org", - app_url: "https://braintrust.dev", - debug_logging: "debug", - }); - - expect(login).toHaveBeenCalledWith({ - apiKey: "test-key", - orgName: "test-org", - appUrl: "https://braintrust.dev", - debugLogLevel: "debug", - }); - }); - - test("loadCLIEnv treats --verbose as a deprecated alias for --debug-logging debug", async () => { - const warnSpy = vi.spyOn(console, "warn").mockImplementation(() => {}); - - await loadCLIEnv({ - verbose: true, - api_key: "test-key", - org_name: "test-org", - app_url: "https://braintrust.dev", - }); - - expect(login).toHaveBeenCalledWith({ - apiKey: "test-key", - orgName: "test-org", - appUrl: "https://braintrust.dev", - debugLogLevel: "debug", - }); - expect(warnSpy).toHaveBeenCalledWith( - "Warning: --verbose is deprecated and will be removed in a future version of braintrust. Use --debug-logging debug to see full stack traces and troubleshooting details.", - ); - }); -}); diff --git a/js/src/cli/functions/infer-source.ts b/js/src/cli/functions/infer-source.ts deleted file mode 100644 index 0d480c621..000000000 --- a/js/src/cli/functions/infer-source.ts +++ /dev/null @@ -1,212 +0,0 @@ -import { SourceMapConsumer } from "source-map"; -import * as fs from "node:fs/promises"; -import { EvaluatorFile, warning } from "../../framework"; -import { loadModule } from "./load-module"; -import { type CodeBundleType as CodeBundle } from "../../generated_types"; -import path from "node:path"; -import type { Node } from "typescript"; - -interface SourceMapContext { - inFiles: Record; - outFileModule: EvaluatorFile; - outFileLines: string[]; - sourceMapDir: string; - sourceMap: SourceMapConsumer; -} - -export async function makeSourceMapContext({ - inFile, - outFile, - sourceMapFile, -}: { - inFile: string; - outFile: string; - sourceMapFile: string; -}): Promise { - const [inFileContents, outFileContents, sourceMap] = await Promise.all([ - fs.readFile(inFile, "utf8"), - fs.readFile(outFile, "utf8"), - (async () => { - const sourceMap = await fs.readFile(sourceMapFile, "utf8"); - const sourceMapJSON = JSON.parse(sourceMap); - return new SourceMapConsumer(sourceMapJSON); - })(), - ]); - return { - inFiles: { [inFile]: inFileContents.split("\n") }, - outFileModule: loadModule({ inFile, moduleText: outFileContents }), - outFileLines: outFileContents.split("\n"), - sourceMapDir: path.dirname(sourceMapFile), - sourceMap, - }; -} - -// eslint-disable-next-line @typescript-eslint/no-unsafe-function-type -function isNative(fn: Function): boolean { - return /\{\s*\[native code\]\s*\}/.test(Function.prototype.toString.call(fn)); -} - -function locationToString(location: CodeBundle["location"]): string { - if (location.type === "experiment") { - return `eval ${location.eval_name} -> ${location.position.type}`; - } else if (location.type === "function") { - return `task ${location.index}`; - } else if (location.type === "sandbox") { - return `sandbox eval ${location.eval_name}`; - } else { - throw new Error(`Unknown location type`); - } -} - -export async function findCodeDefinition({ - location, - ctx: { inFiles, outFileModule, outFileLines, sourceMapDir, sourceMap }, -}: { - location: CodeBundle["location"]; - ctx: SourceMapContext; -}): Promise { - // eslint-disable-next-line @typescript-eslint/no-unsafe-function-type - let fn: Function | undefined = undefined; - - if (location.type === "experiment" || location.type === "sandbox") { - const evaluator = outFileModule.evaluators[location.eval_name]?.evaluator; - if (!evaluator) { - // eslint-disable-next-line no-restricted-properties -- preserving intentional console usage. - console.warn( - warning( - `Warning: failed to find evaluator for ${location.eval_name}. Will not display preview.`, - ), - ); - return undefined; - } - - if (location.type === "sandbox") { - fn = evaluator.task; - } else { - fn = - location.position.type === "task" - ? evaluator.task - : location.position.type === "scorer" - ? (evaluator.scores ?? [])[location.position.index] - : (evaluator.classifiers ?? [])[location.position.index]; - } - } else if (location.type === "function") { - fn = outFileModule.functions[location.index].handler; - } else { - throw new Error(`Unknown location type`); - } - - if (!fn) { - // eslint-disable-next-line no-restricted-properties -- preserving intentional console usage. - console.warn( - warning( - `Warning: failed to find ${locationToString(location)}. Will not display preview.`, - ), - ); - return undefined; - } - - const sourceCode = fn.toString(); - if (isNative(fn)) { - return undefined; - } - let lineNumber = 0; - let columnNumber = -1; - for (const line of outFileLines) { - const sourceDefinition = line.indexOf(sourceCode); - if (sourceDefinition !== -1) { - columnNumber = sourceDefinition; - break; - } - lineNumber++; - } - - if (columnNumber === -1) { - // eslint-disable-next-line no-restricted-properties -- preserving intentional console usage. - console.warn( - warning( - `Warning: failed to find code definition for ${fn.name}. Will not display preview.`, - ), - ); - return undefined; - } - const originalPosition = sourceMap.originalPositionFor({ - line: lineNumber + 1, - column: columnNumber + 1, - }); - - if (originalPosition.source === null || originalPosition.line === null) { - return undefined; - } - - if (!inFiles[originalPosition.source]) { - const originalFile = path.join(sourceMapDir, originalPosition.source); - inFiles[originalPosition.source] = ( - await fs.readFile(originalFile, "utf-8") - ).split("\n"); - } - - const originalLines = inFiles[originalPosition.source]; - - // Parse the file with Typescript to find the function definition - const ts = await getTsModule(); - if (!ts) { - return undefined; - } - const sourceFile = ts.createSourceFile( - originalPosition.source, - originalLines.join("\n"), - ts.ScriptTarget.Latest, - true, - ); - let functionNode: Node | undefined = undefined; - const targetPosition = ts.getPositionOfLineAndCharacter( - sourceFile, - originalPosition.line - 1, - originalPosition.column || 0, - ); - - ts.forEachChild(sourceFile, function visit(node) { - if (node.pos <= targetPosition && targetPosition < node.end) { - if ( - ts.isFunctionDeclaration(node) || - ts.isFunctionExpression(node) || - ts.isArrowFunction(node) - ) { - functionNode = node; - } else { - ts.forEachChild(node, visit); - } - } - }); - - if (!functionNode) { - return undefined; - } - - const printer = ts.createPrinter(); - const functionDefinition = printer.printNode( - ts.EmitHint.Unspecified, - functionNode, - sourceFile, - ); - - return functionDefinition; -} - -let tsModule: typeof import("typescript") | undefined = undefined; -async function getTsModule() { - if (!tsModule) { - try { - tsModule = require("typescript"); - } catch { - // eslint-disable-next-line no-restricted-properties -- preserving intentional console usage. - console.warn( - warning( - "Failed to load TypeScript module. Will not use TypeScript to derive preview.", - ), - ); - } - } - return tsModule; -} diff --git a/js/src/cli/functions/load-module.ts b/js/src/cli/functions/load-module.ts deleted file mode 100644 index 1fa7296b0..000000000 --- a/js/src/cli/functions/load-module.ts +++ /dev/null @@ -1,54 +0,0 @@ -import nodeModulesPaths from "../jest/nodeModulesPaths"; -import path, { dirname } from "node:path"; -import { _internalGetGlobalState } from "../../logger"; -import { EvaluatorFile } from "../../framework"; -import { applyAutoInstrumentation } from "../../node/apply-auto-instrumentation"; -import { debugLogger } from "../../debug-logger"; - -function evalWithModuleContext(inFile: string, evalFn: () => T): T { - const modulePaths = [...module.paths]; - try { - module.paths = nodeModulesPaths(path.dirname(inFile), {}); - return evalFn(); - } finally { - module.paths = modulePaths; - } -} - -export function loadModule({ - inFile, - moduleText, -}: { - inFile: string; - moduleText: string; -}): EvaluatorFile { - return evalWithModuleContext(inFile, () => { - globalThis._evals = { - functions: [], - prompts: [], - parameters: [], - evaluators: {}, - reporters: {}, - }; - globalThis._lazy_load = true; - const state = _internalGetGlobalState(); - (globalThis as any)[Symbol.for("braintrust-state")] = state; - const __filename = inFile; - const __dirname = dirname(__filename); - try { - applyAutoInstrumentation(); - } catch (error) { - debugLogger.warn( - "Failed to enable auto-instrumentation for external eval dependencies; bundled dependencies remain instrumented:", - error, - ); - } - new Function("require", "module", "__filename", "__dirname", moduleText)( - require, - module, - __filename, - __dirname, - ); - return { ...globalThis._evals }; - }); -} diff --git a/js/src/cli/functions/upload.test.ts b/js/src/cli/functions/upload.test.ts deleted file mode 100644 index f0ecde997..000000000 --- a/js/src/cli/functions/upload.test.ts +++ /dev/null @@ -1,119 +0,0 @@ -import { describe, expect, test, vi } from "vitest"; -import { buildBundledFunctionEntry } from "./upload"; - -vi.mock("./infer-source", () => ({ - findCodeDefinition: vi.fn(), - makeSourceMapContext: vi.fn(), -})); - -describe("buildBundledFunctionEntry", () => { - test("preserves tags and existing function fields", async () => { - const spec = { - project_id: "proj-123", - name: "test-tool", - slug: "test-tool", - description: "Test tool", - location: { - type: "function" as const, - index: 0, - }, - function_type: "tool" as const, - origin: { - object_type: "experiment" as const, - object_id: "exp-123", - internal: false, - }, - function_schema: { - parameters: { type: "object" }, - returns: { type: "string" }, - }, - if_exists: "replace" as const, - tags: ["ci", "sdk"], - metadata: { owner: "sdk" }, - }; - - const entry = await buildBundledFunctionEntry({ - spec, - runtime_context: { runtime: "node", version: "22.0.0" }, - bundleId: "bundle-123", - sourceMapContext: undefined, - }); - - expect(entry).toMatchObject({ - project_id: "proj-123", - name: "test-tool", - slug: "test-tool", - description: "Test tool", - origin: spec.origin, - function_type: "tool", - function_schema: spec.function_schema, - if_exists: "replace", - tags: ["ci", "sdk"], - metadata: { owner: "sdk" }, - function_data: { - type: "code", - data: { - type: "bundle", - runtime_context: { runtime: "node", version: "22.0.0" }, - location: spec.location, - bundle_id: "bundle-123", - }, - }, - }); - expect(((entry.function_data as any).data as any).preview).toBeUndefined(); - }); - - test("does not invent tags when they are omitted", async () => { - const entry = await buildBundledFunctionEntry({ - spec: { - project_id: "proj-123", - name: "test-tool", - slug: "test-tool", - description: "Test tool", - location: { - type: "function" as const, - index: 0, - }, - function_type: "tool" as const, - }, - runtime_context: { runtime: "node", version: "22.0.0" }, - bundleId: "bundle-123", - sourceMapContext: undefined, - }); - - expect(entry.tags).toBeUndefined(); - }); - - test("preserves classifier experiment locations", async () => { - const entry = await buildBundledFunctionEntry({ - spec: { - project_id: "proj-123", - name: "test-classifier", - slug: "test-classifier", - description: "Test classifier", - location: { - type: "experiment" as const, - eval_name: "eval-1", - position: { - type: "classifier" as const, - index: 0, - }, - }, - function_type: "classifier" as const, - }, - runtime_context: { runtime: "node", version: "22.0.0" }, - bundleId: "bundle-123", - sourceMapContext: undefined, - }); - - expect(entry.function_type).toBe("classifier"); - expect(((entry.function_data as any).data as any).location).toEqual({ - type: "experiment", - eval_name: "eval-1", - position: { - type: "classifier", - index: 0, - }, - }); - }); -}); diff --git a/js/src/cli/functions/upload.ts b/js/src/cli/functions/upload.ts deleted file mode 100644 index da513f503..000000000 --- a/js/src/cli/functions/upload.ts +++ /dev/null @@ -1,516 +0,0 @@ -import { - type CodeBundleType as CodeBundle, - type FunctionType as FunctionObject, - type IfExistsType as IfExists, -} from "../../generated_types"; -import type { BuildSuccess, FileHandle } from "../types"; -import { classifierName, scorerName, warning } from "../../framework"; -import { - _internalGetGlobalState, - Experiment, - FailedHTTPResponse, -} from "../../logger"; -import * as esbuild from "esbuild"; -import fs from "node:fs"; -import path from "node:path"; -import { createGzip } from "node:zlib"; -import { addAzureBlobHeaders, isEmpty } from "../../util"; -import { z } from "zod/v3"; -import { capitalize } from "../../../util/index"; -import { findCodeDefinition, makeSourceMapContext } from "./infer-source"; -import { slugify } from "../../../util/string_util"; -import { zodToJsonSchema } from "../../zod/utils"; -import pluralize from "pluralize"; -import { - FunctionEvent, - ProjectNameIdMap, - serializeRemoteEvalParametersContainer, -} from "../../framework2"; - -interface BundledFunctionSpec { - project_id: string; - name: string; - slug: string; - description: string; - location: CodeBundle["location"]; - function_type: FunctionObject["function_type"]; - origin?: FunctionObject["origin"]; - function_schema?: FunctionObject["function_schema"]; - if_exists?: IfExists; - tags?: string[]; - metadata?: Record; -} - -type BundledFunctionEntry = FunctionEvent & { - origin?: FunctionObject["origin"]; - function_schema?: FunctionObject["function_schema"]; -}; - -const SANDBOX_GROUP_NAME_METADATA_KEY = "_bt_sandbox_group_name"; - -const pathInfoSchema = z - .strictObject({ - url: z.string(), - bundleId: z.string(), - }) - .strip(); - -export async function uploadHandleBundles({ - buildResults, - evalToExperiment, - bundlePromises, - handles, - setCurrent, - showDetailedErrors, - defaultIfExists, -}: { - buildResults: BuildSuccess[]; - evalToExperiment?: Record>; - bundlePromises: { - [k: string]: Promise>; - }; - handles: Record; - showDetailedErrors: boolean; - setCurrent: boolean; - defaultIfExists: IfExists; -}) { - // eslint-disable-next-line no-restricted-properties -- preserving intentional console usage. - console.error( - `Processing ${buildResults.length} ${pluralize("file", buildResults.length)}...`, - ); - - const projectNameToId = new ProjectNameIdMap(); - - const uploadPromises = buildResults.map(async (result) => { - if (result.type !== "success") { - return; - } - const sourceFile = result.sourceFile; - - const bundleSpecs: BundledFunctionSpec[] = []; - const prompts: FunctionEvent[] = []; - - if (setCurrent) { - for (let i = 0; i < result.evaluator.functions.length; i++) { - const fn = result.evaluator.functions[i]; - const project_id = await projectNameToId.resolve(fn.project); - - bundleSpecs.push({ - project_id: project_id, - name: fn.name, - slug: fn.slug, - description: fn.description ?? "", - function_type: fn.type, - location: { - type: "function", - index: i, - }, - function_schema: - fn.parameters || fn.returns - ? { - parameters: fn.parameters - ? zodToJsonSchema(fn.parameters) - : undefined, - returns: fn.returns ? zodToJsonSchema(fn.returns) : undefined, - } - : undefined, - if_exists: fn.ifExists, - tags: fn.tags, - metadata: fn.metadata, - }); - } - - for (const prompt of result.evaluator.prompts) { - prompts.push(await prompt.toFunctionDefinition(projectNameToId)); - } - - if (result.evaluator.parameters != null) { - for (const param of result.evaluator.parameters) { - prompts.push(await param.toFunctionDefinition(projectNameToId)); - } - } - } - - for (const evaluator of Object.values(result.evaluator.evaluators)) { - const experiment = - evalToExperiment?.[sourceFile]?.[evaluator.evaluator.evalName]; - - const baseInfo = { - project_id: experiment - ? (await experiment.project).id - : await projectNameToId.getId(evaluator.evaluator.projectName), - }; - - const namePrefix = setCurrent - ? evaluator.evaluator.experimentName - ? `${evaluator.evaluator.experimentName}` - : evaluator.evaluator.evalName - : experiment - ? `${await experiment.name}` - : evaluator.evaluator.evalName; - - const experimentId = experiment ? await experiment.id : undefined; - const origin: FunctionObject["origin"] = experimentId - ? { - object_type: "experiment", - object_id: experimentId, - internal: !setCurrent, - } - : undefined; - - const fileSpecs: BundledFunctionSpec[] = [ - { - ...baseInfo, - // There is a very small chance that someone names a function with the same convention, but - // let's assume it's low enough that it doesn't matter. - ...formatNameAndSlug(["eval", namePrefix, "task"]), - description: `Task for eval ${namePrefix}`, - location: { - type: "experiment", - eval_name: evaluator.evaluator.evalName, - position: { type: "task" }, - }, - function_type: "task", - origin, - }, - ...(evaluator.evaluator.scores ?? []).map( - (score, i): BundledFunctionSpec => { - const name = scorerName(score, i); - return { - ...baseInfo, - // There is a very small chance that someone names a function with the same convention, but - // let's assume it's low enough that it doesn't matter. - ...formatNameAndSlug(["eval", namePrefix, "scorer", name]), - description: `Score ${name} for eval ${namePrefix}`, - location: { - type: "experiment", - eval_name: evaluator.evaluator.evalName, - position: { type: "scorer", index: i }, - }, - function_type: "scorer", - origin, - }; - }, - ), - ...(evaluator.evaluator.classifiers ?? []).map( - (classifier, i): BundledFunctionSpec => { - const name = classifierName(classifier, i); - return { - ...baseInfo, - ...formatNameAndSlug(["eval", namePrefix, "classifier", name]), - description: `Classifier ${name} for eval ${namePrefix}`, - location: { - type: "experiment", - eval_name: evaluator.evaluator.evalName, - position: { type: "classifier", index: i }, - }, - function_type: "classifier", - origin, - }; - }, - ), - ]; - - bundleSpecs.push(...fileSpecs); - - if (setCurrent) { - const sourceStem = path - .basename(sourceFile, path.extname(sourceFile)) - .replace(/\.eval$/, ""); - const evalName = evaluator.evaluator.evalName; - const sandboxGroupName = sourceStem; - - const resolvedParameters = evaluator.evaluator.parameters - ? await Promise.resolve(evaluator.evaluator.parameters) - : undefined; - - const evaluatorDefinition = { - ...(resolvedParameters - ? { - parameters: - serializeRemoteEvalParametersContainer(resolvedParameters), - } - : {}), - scores: (evaluator.evaluator.scores ?? []).map((score, i) => ({ - name: scorerName(score, i), - })), - classifiers: (evaluator.evaluator.classifiers ?? []).map( - (classifier, i) => ({ - name: classifierName(classifier, i), - }), - ), - }; - - bundleSpecs.push({ - ...baseInfo, - name: `Eval ${evalName} sandbox`, - slug: slugify(`${sourceStem}-${evalName}-sandbox`), - description: `Sandbox eval ${evalName}`, - location: { - type: "sandbox", - sandbox_spec: { - provider: "lambda", - }, - entrypoints: [sourceFile], - eval_name: evalName, - evaluator_definition: evaluatorDefinition, - }, - function_type: "sandbox", - metadata: { - [SANDBOX_GROUP_NAME_METADATA_KEY]: sandboxGroupName, - }, - origin, - }); - } - } - - const slugs: Set = new Set(); - for (const spec of bundleSpecs) { - if (slugs.has(spec.slug)) { - throw new Error(`Duplicate slug: ${spec.slug}`); - } - slugs.add(spec.slug); - } - for (const prompt of prompts) { - if (slugs.has(prompt.slug)) { - throw new Error(`Duplicate slug: ${prompt.slug}`); - } - slugs.add(prompt.slug); - } - - return await uploadBundles({ - sourceFile, - prompts, - bundleSpecs, - bundlePromises, - handles, - defaultIfExists, - showDetailedErrors, - }); - }); - - const uploadResults = await Promise.all(uploadPromises); - const numUploaded = uploadResults.length; - const numFailed = uploadResults.filter((result) => !result).length; - - // eslint-disable-next-line no-restricted-properties -- preserving intentional console usage. - console.error( - `${numUploaded} ${pluralize("file", numUploaded)} uploaded ${ - numFailed > 0 - ? `with ${numFailed} error${numFailed > 1 ? "s" : ""}` - : "successfully" - }.`, - ); - - return { - numTotal: buildResults.length, - numUploaded, - numFailed, - }; -} - -async function uploadBundles({ - sourceFile, - prompts, - bundleSpecs, - bundlePromises, - handles, - defaultIfExists, - showDetailedErrors, -}: { - sourceFile: string; - prompts: FunctionEvent[]; - bundleSpecs: BundledFunctionSpec[]; - bundlePromises: { - [k: string]: Promise>; - }; - handles: Record; - defaultIfExists: IfExists; - showDetailedErrors: boolean; -}): Promise { - const orgId = _internalGetGlobalState().orgId; - if (!orgId) { - throw new Error("No organization ID found"); - } - - const loggerConn = _internalGetGlobalState().apiConn(); - const runtime_context = { - runtime: "node", - version: process.version.slice(1), - } as const; - - const bundle = await bundlePromises[sourceFile]; - const bundleFileName = handles[sourceFile].bundleFile; - if (!bundle || !bundleFileName) { - return false; - } - - const sourceMapContextPromise = makeSourceMapContext({ - inFile: sourceFile, - outFile: bundleFileName, - sourceMapFile: bundleFileName + ".map", - }); - - let pathInfo: z.infer | undefined = undefined; - if (bundleSpecs.length > 0) { - try { - pathInfo = pathInfoSchema.parse( - await loggerConn.post_json("function/code", { - org_id: orgId, - runtime_context, - }), - ); - } catch (e) { - if (showDetailedErrors) { - // eslint-disable-next-line no-restricted-properties -- preserving intentional console usage. - console.error(e); - } - const msg = - e instanceof FailedHTTPResponse - ? `Unable to upload your code. ${e.status} (${e.text}): ${e.data}` - : `Unable to upload your code. You most likely need to update the API: ${e}`; - // eslint-disable-next-line no-restricted-properties -- preserving intentional console usage. - console.error(warning(msg)); - return false; - } - } - - // Upload bundleFile to pathInfo.url - if (isEmpty(bundleFileName)) { - throw new Error("No bundle file found"); - } - const bundleFile = path.resolve(bundleFileName); - const uploadPromise = (async (): Promise => { - if (!pathInfo) { - return true; - } - const bundleStream = fs.createReadStream(bundleFile).pipe(createGzip()); - const bundleData = await new Promise((resolve, reject) => { - const chunks: Buffer[] = []; - bundleStream.on("data", (chunk) => { - chunks.push(chunk); - }); - bundleStream.on("end", () => { - resolve(Buffer.concat(chunks)); - }); - bundleStream.on("error", reject); - }); - - const headers = { - "Content-Encoding": "gzip", - }; - addAzureBlobHeaders(headers, pathInfo.url); - - const resp = await fetch(pathInfo.url, { - method: "PUT", - body: bundleData, - headers, - }); - if (!resp.ok) { - throw new Error( - `Failed to upload bundle: ${resp.status} ${await resp.text()}`, - ); - } - return true; - })(); - - const sourceMapContext = await sourceMapContextPromise; - - // Insert the spec as prompt data - const functionEntries: FunctionEvent[] = [ - ...prompts, - // eslint-disable-next-line @typescript-eslint/consistent-type-assertions - ...((await Promise.all( - bundleSpecs.map((spec) => - buildBundledFunctionEntry({ - spec, - runtime_context, - bundleId: pathInfo!.bundleId, - sourceMapContext, - }), - ), - )) as FunctionEvent[]), - ].map((fn) => ({ - ...fn, - if_exists: fn.if_exists ?? defaultIfExists, - })); - - const logPromise = (async (): Promise => { - try { - await _internalGetGlobalState().apiConn().post_json("insert-functions", { - functions: functionEntries, - }); - } catch (e) { - if (showDetailedErrors) { - // eslint-disable-next-line no-restricted-properties -- preserving intentional console usage. - console.error(e); - } - const msg = - e instanceof FailedHTTPResponse - ? `Failed to save function definitions for '${sourceFile}'. ${e.status} (${e.text}): ${e.data}` - : `Failed to save function definitions for '${sourceFile}'. You most likely need to update the API: ${e}`; - // eslint-disable-next-line no-restricted-properties -- preserving intentional console usage. - console.warn(warning(msg)); - return false; - } - return true; - })(); - - const [uploadSuccess, logSuccess] = await Promise.all([ - uploadPromise, - logPromise, - ]); - - return uploadSuccess && logSuccess; -} - -function formatNameAndSlug(pieces: string[]) { - const nonEmptyPieces = pieces.filter((piece) => piece.trim() !== ""); - return { - name: capitalize(nonEmptyPieces.join(" ")), - slug: slugify(nonEmptyPieces.join("-")), - }; -} - -export async function buildBundledFunctionEntry({ - spec, - runtime_context, - bundleId, - sourceMapContext, -}: { - spec: BundledFunctionSpec; - runtime_context: { - runtime: "node"; - version: string; - }; - bundleId: string; - sourceMapContext?: Awaited>; -}): Promise { - return { - project_id: spec.project_id, - name: spec.name, - slug: spec.slug, - description: spec.description, - function_data: { - type: "code", - data: { - type: "bundle", - runtime_context, - location: spec.location, - bundle_id: bundleId, - preview: sourceMapContext - ? await findCodeDefinition({ - location: spec.location, - ctx: sourceMapContext, - }) - : undefined, - }, - }, - origin: spec.origin, - function_type: spec.function_type ?? undefined, - function_schema: spec.function_schema, - if_exists: spec.if_exists, - tags: spec.tags, - metadata: spec.metadata, - }; -} diff --git a/js/src/cli/index.ts b/js/src/cli/index.ts deleted file mode 100755 index eb5004555..000000000 --- a/js/src/cli/index.ts +++ /dev/null @@ -1,1194 +0,0 @@ -#!/usr/bin/env node - -import * as esbuild from "esbuild"; -import * as dotenv from "dotenv"; -import fs from "node:fs"; -import os from "node:os"; -import path from "node:path"; -import util, { styleText } from "node:util"; -import * as fsWalk from "@nodelib/fs.walk"; -import { minimatch } from "minimatch"; -import { ArgumentParser } from "argparse"; -import { v4 as uuidv4 } from "uuid"; -import pluralize from "pluralize"; -import { - login, - init as _initExperiment, - Experiment, - BaseMetadata, - Dataset, - type ParametersRef, - RemoteEvalParameters, -} from "../logger"; -import type { ProgressReporter } from "../reporters/types"; -import { - BarProgressReporter, - SimpleProgressReporter, -} from "./reporters/progress"; -import { terminalLink } from "termi-link"; - -// Re-use the module resolution logic from Jest -import type { ReporterDef } from "../reporters/types"; -import { - EvalData, - EvaluatorDef, - EvaluatorFile, - Filter, - callEvaluatorData, - error, - logError, - parseFilters, - runEvaluator, -} from "../framework"; -import { fancyReporter, warning } from "./reporters/eval"; -import { configureNode } from "../node/config"; -import { isEmpty } from "../util"; -import { loadEnvConfig } from "@next/env"; -import type { - BuildSuccess, - BuildFailure, - BtBuildResult, - FileHandle, - EvaluatorState, -} from "./types"; -import { uploadHandleBundles } from "./functions/upload"; -import { loadModule } from "./functions/load-module"; -import { bundleCommand } from "./util/bundle"; -import { RunArgs } from "./util/types"; -import { - normalizeDebugLoggingArgs, - shouldShowDetailedErrors, -} from "./util/debug-logging"; -import { pullCommand } from "./util/pull"; -import { runDevServer } from "../../dev/server"; -import { braintrustEsbuildPlugin } from "../auto-instrumentations/bundler/esbuild"; - -// This requires require -// https://stackoverflow.com/questions/50822310/how-to-import-package-json-in-typescript -const { version } = require("../../package.json"); - -// TODO: This could be loaded from configuration -const INCLUDE_EVAL = [ - "**/*.eval.ts", - "**/*.eval.tsx", - "**/*.eval.js", - "**/*.eval.jsx", -]; -const INCLUDE_BUNDLE = ["**/*.ts", "**/*.tsx", "**/*.js", "**/*.jsx"]; -const EXCLUDE = ["**/node_modules/**", "**/dist/**", "**/build/**"]; -const OUT_EXT = "js"; - -configureNode(); - -function evaluateBuildResults( - inFile: string, - buildResult: esbuild.BuildResult, -): EvaluatorFile | null { - if (!buildResult.outputFiles) { - return null; - } - const moduleText = buildResult.outputFiles[0].text; - return loadModule({ inFile, moduleText }); -} - -async function initExperiment( - evaluator: EvaluatorDef, - evaluatorData: { - data: EvalData; - baseExperiment: string | undefined; - }, -) { - const { data, baseExperiment: defaultBaseExperiment } = evaluatorData; - const parameters = await getExperimentParametersRef(evaluator.parameters); - // NOTE: This code is duplicated with initExperiment in js/src/framework.ts. - // Make sure to update that if you change this. - const logger = _initExperiment({ - state: evaluator.state, - ...(evaluator.projectId - ? { projectId: evaluator.projectId } - : { project: evaluator.projectName }), - experiment: evaluator.experimentName, - description: evaluator.description, - metadata: evaluator.metadata, - tags: evaluator.tags, - isPublic: evaluator.isPublic, - update: evaluator.update, - baseExperiment: evaluator.baseExperimentName ?? defaultBaseExperiment, - baseExperimentId: evaluator.baseExperimentId, - gitMetadataSettings: evaluator.gitMetadataSettings, - repoInfo: evaluator.repoInfo, - dataset: Dataset.isDataset(data) ? data : undefined, - parameters, - setCurrent: false, - }); - const info = await logger.summarize({ summarizeScores: false }); - const linkText = info.experimentUrl - ? terminalLink(info.experimentUrl, info.experimentUrl, { - fallback: (_text: string, url: string) => url, - }) - : "locally"; - // eslint-disable-next-line no-restricted-properties -- preserving intentional console usage. - console.error( - styleText("cyan", "▶") + - ` Experiment ${styleText("bold", info.experimentName)} is running at ${linkText}`, - ); - return logger; -} - -async function getExperimentParametersRef( - parameters: - | Record - | RemoteEvalParameters - | Promise> - | undefined, -): Promise { - if (!parameters) { - return undefined; - } - - const resolvedParameters = - parameters instanceof Promise ? await parameters : parameters; - - if (!RemoteEvalParameters.isParameters(resolvedParameters)) { - return undefined; - } - - if (resolvedParameters.id === undefined) { - return undefined; - } - - return { - id: resolvedParameters.id, - version: resolvedParameters.version, - }; -} - -function resolveReporter( - reporter: string | ReporterDef | undefined, - reporters: Record>, -) { - if (typeof reporter === "string") { - if (!reporters[reporter]) { - throw new Error(`Reporter ${reporter} not found`); - } - return reporters[reporter]; - } else if (!isEmpty(reporter)) { - return reporter; - } else if (Object.keys(reporters).length === 0) { - return fancyReporter; - } else if (Object.keys(reporters).length === 1) { - return reporters[Object.keys(reporters)[0]]; - } else { - const reporterNames = Object.keys(reporters).join(", "); - throw new Error( - `Multiple reporters found (${reporterNames}). Please specify a reporter explicitly.`, - ); - } -} - -type AllReports = Record< - string, - { - reporter: ReporterDef; - results: (unknown | Promise)[]; - } ->; - -function addReport( - evalReports: AllReports, - reporter: ReporterDef, - report: unknown, -) { - if (!evalReports[reporter.name]) { - evalReports[reporter.name] = { - reporter, - results: [], - }; - } - evalReports[reporter.name].results.push(report); -} - -function buildWatchPluginForEvaluator( - inFile: string, - opts: EvaluatorOpts, -): esbuild.Plugin { - const evaluators: EvaluatorState = { - evaluators: [], - reporters: {}, - }; - const plugin = { - name: "run-evalutator-on-end", - setup(build: esbuild.PluginBuild) { - build.onEnd(async (result) => { - // eslint-disable-next-line no-restricted-properties -- preserving intentional console usage. - console.error(`Done building ${inFile}`); - - if (!result.outputFiles) { - if (opts.showDetailedErrors) { - // eslint-disable-next-line no-restricted-properties -- preserving intentional console usage. - console.warn(`Failed to compile ${inFile}`); - // eslint-disable-next-line no-restricted-properties -- preserving intentional console usage. - console.warn(result.errors); - } else { - // eslint-disable-next-line no-restricted-properties -- preserving intentional console usage. - console.warn(`Failed to compile ${inFile}: ${result.errors}`); - } - return; - } - - const evalResult = evaluateBuildResults(inFile, result); - if (!evalResult) { - return; - } - - evaluators.evaluators = evaluators.evaluators.filter( - (e) => e.sourceFile !== inFile, - ); - - // Update the evaluators and reporters - for (const evaluator of Object.values(evalResult.evaluators)) { - evaluators.evaluators.push({ - sourceFile: inFile, - // eslint-disable-next-line @typescript-eslint/consistent-type-assertions - evaluator: evaluator.evaluator as EvaluatorDef< - unknown, - unknown, - unknown, - BaseMetadata - >, - reporter: evaluator.reporter, - }); - } - for (const [reporterName, reporter] of Object.entries( - evalResult.reporters, - )) { - evaluators.reporters[reporterName] = reporter; - } - - const evalReports: Record< - string, - { - reporter: ReporterDef; - results: unknown[]; - } - > = {}; - for (const evaluatorDef of Object.values(evalResult.evaluators)) { - const { evaluator, reporter } = evaluatorDef; - const evalData = callEvaluatorData(evaluator.data); - const logger = opts.noSendLogs - ? null - : await initExperiment(evaluator, evalData); - const evaluatorResult = await runEvaluator( - logger, - { - ...evaluator, - data: evalData.data, - }, - opts.progressReporter, - opts.filters, - undefined, - undefined, - ); - const resolvedReporter = resolveReporter( - reporter, - evaluators.reporters, // Let these accumulate across all files. - ); - - const report = resolvedReporter.reportEval( - evaluator, - evaluatorResult, - { - verbose: opts.showDetailedErrors, - jsonl: opts.jsonl, - }, - ); - - addReport(evalReports, resolvedReporter, report); - } - - for (const [reporterName, { reporter, results }] of Object.entries( - evalReports, - )) { - const success = await reporter.reportRun(await Promise.all(results)); - if (!success) { - // eslint-disable-next-line no-restricted-properties -- preserving intentional console usage. - console.error(error(`Reporter ${reporterName} failed.`)); - } - } - }); - }, - }; - - return plugin; -} - -async function initFile({ - inFile, - outFile, - bundleFile, - tsconfig, - plugins, - externalPackages, -}: { - inFile: string; - outFile: string; - bundleFile: string; - tsconfig?: string; - plugins?: PluginMaker[]; - externalPackages?: string[]; -}): Promise { - const buildOptions = buildOpts({ - fileName: inFile, - outFile, - tsconfig, - plugins, - externalPackages, - }); - const ctx = await esbuild.context(buildOptions); - - return { - inFile, - outFile, - bundleFile, - rebuild: async () => { - try { - const result = await ctx.rebuild(); - if (!result.outputFiles) { - return { - type: "failure", - error: new Error("No output file generated"), - sourceFile: inFile, - }; - } - const evaluator = evaluateBuildResults(inFile, result) || { - functions: [], - prompts: [], - parameters: [], - evaluators: {}, - reporters: {}, - }; - return { type: "success", result, evaluator, sourceFile: inFile }; - } catch (e) { - // eslint-disable-next-line @typescript-eslint/consistent-type-assertions - return { type: "failure", error: e as Error, sourceFile: inFile }; - } - }, - bundle: async () => { - const buildOptions: esbuild.BuildOptions = { - ...buildOpts({ - fileName: inFile, - outFile: bundleFile, - tsconfig, - plugins: [], - externalPackages, - }), - external: ["fsevents", "chokidar"], - write: true, - minify: true, - sourcemap: true, - }; - return await esbuild.build(buildOptions); - }, - watch: () => { - ctx.watch(); - }, - destroy: async () => { - await ctx.dispose(); - }, - }; -} - -interface EvaluatorOpts { - showDetailedErrors: boolean; - apiKey?: string; - orgName?: string; - appUrl?: string; - noSendLogs: boolean; - bundle: boolean; - setCurrent: boolean; - terminateOnFailure: boolean; - watch: boolean; - list: boolean; - jsonl: boolean; - filters: Filter[]; - progressReporter: ProgressReporter; -} - -export function handleBuildFailure({ - result, - terminateOnFailure, - showDetailedErrors, -}: { - result: BuildFailure; - terminateOnFailure: boolean; - showDetailedErrors: boolean; -}) { - if (terminateOnFailure) { - throw result.error; - } else if (showDetailedErrors) { - // eslint-disable-next-line no-restricted-properties -- preserving intentional console usage. - console.warn(`Failed to compile ${result.sourceFile}`); - // eslint-disable-next-line no-restricted-properties -- preserving intentional console usage. - console.warn(result.error); - } else { - // eslint-disable-next-line no-restricted-properties -- preserving intentional console usage. - console.warn( - `Failed to compile ${result.sourceFile}: ${result.error.message}`, - ); - } -} - -function updateEvaluators( - evaluators: EvaluatorState, - buildResults: BtBuildResult[], - opts: EvaluatorOpts, -) { - for (const result of buildResults) { - if (result.type === "failure") { - handleBuildFailure({ - result, - terminateOnFailure: opts.terminateOnFailure, - showDetailedErrors: opts.showDetailedErrors, - }); - continue; - } - - for (const evaluator of Object.values(result.evaluator.evaluators)) { - evaluators.evaluators.push({ - sourceFile: result.sourceFile, - // eslint-disable-next-line @typescript-eslint/consistent-type-assertions - evaluator: evaluator.evaluator as EvaluatorDef< - unknown, - unknown, - unknown, - BaseMetadata - >, - reporter: evaluator.reporter, - }); - } - - for (const [reporterName, reporter] of Object.entries( - result.evaluator.reporters, - )) { - if ( - evaluators.reporters[reporterName] && - evaluators.reporters[reporterName] !== reporter - ) { - // eslint-disable-next-line no-restricted-properties -- preserving intentional console usage. - console.warn( - warning( - `Reporter '${reporterName}' already exists. Will skip '${reporterName}' from ${result.sourceFile}.`, - ), - ); - continue; - } - evaluators.reporters[reporterName] = reporter; - } - } -} - -async function runAndWatch({ - handles, - onExit, -}: { - handles: Record; - onExit?: () => void; -}) { - const count = Object.keys(handles).length; - // eslint-disable-next-line no-restricted-properties -- preserving intentional console usage. - console.error(`Watching ${pluralize("file", count, true)}...`); - - Object.values(handles).map((handle) => handle.watch()); - - ["SIGINT", "SIGTERM"].forEach((signal: string) => { - process.on(signal, function () { - // eslint-disable-next-line no-restricted-properties -- preserving intentional console usage. - console.error("Stopped watching."); - for (const handle of Object.values(handles)) { - handle.destroy(); - } - onExit?.(); - process.exit(0); - }); - }); - - // Wait forever while we watch. - await new Promise(() => {}); -} - -export async function buildEvaluators( - handles: Record, - opts: EvaluatorOpts, -): Promise<{ evaluators: EvaluatorState; buildResults: BtBuildResult[] }> { - const buildPromises = Object.values(handles).map((handle) => - handle.rebuild(), - ); - - const buildResults = await Promise.all(buildPromises); - - const evaluators: EvaluatorState = { - evaluators: [], - reporters: {}, - }; - updateEvaluators(evaluators, buildResults, opts); - return { evaluators, buildResults }; -} - -async function runOnce( - handles: Record, - opts: EvaluatorOpts, -) { - const bundlePromises = opts.bundle - ? Object.fromEntries( - Object.entries(handles).map(([inFile, handle]) => [ - inFile, - handle.bundle(), - ]), - ) - : null; - - const { evaluators, buildResults } = await buildEvaluators(handles, opts); - - if (opts.list) { - for (const evaluator of evaluators.evaluators) { - // eslint-disable-next-line no-restricted-properties -- preserving intentional console usage. - console.log(evaluator.evaluator.evalName); - } - return true; - } - - // map from file name -> eval name -> experiment - const evalToExperiment: Record> = {}; - - const resultPromises = evaluators.evaluators.map(async (evaluator) => { - const evalData = callEvaluatorData(evaluator.evaluator.data); - // TODO: For now, use the eval name as the project. However, we need to evolve - // the definition of a project and create a new concept called run, so that we - // can name the experiment/evaluation within the run the evaluator's name. - const logger = opts.noSendLogs - ? null - : await initExperiment(evaluator.evaluator, evalData); - try { - return await runEvaluator( - logger, - { - ...evaluator.evaluator, - data: evalData.data, - }, - opts.progressReporter, - opts.filters, - undefined, - undefined, - ); - } finally { - if (logger) { - if (!evalToExperiment[evaluator.sourceFile]) { - evalToExperiment[evaluator.sourceFile] = {}; - } - evalToExperiment[evaluator.sourceFile][evaluator.evaluator.evalName] = - logger; - - await logger.flush(); - } - } - }); - - // eslint-disable-next-line no-restricted-properties -- preserving intentional console usage. - console.error( - styleText( - "dim", - `Processing ${styleText("bold", String(resultPromises.length))} evaluator${resultPromises.length === 1 ? "" : "s"}...`, - ), - ); - const allEvalsResults = await Promise.all(resultPromises); - opts.progressReporter.stop(); - // eslint-disable-next-line no-restricted-properties -- preserving intentional console usage. - console.error(""); - - const evalReports: Record< - string, - { - reporter: ReporterDef; - results: unknown[]; - } - > = {}; - for (let idx = 0; idx < evaluators.evaluators.length; idx++) { - const evaluator = evaluators.evaluators[idx]; - const resolvedReporter = resolveReporter( - evaluator.reporter, - evaluators.reporters, - ); - - const report = resolvedReporter.reportEval( - evaluator.evaluator, - // eslint-disable-next-line @typescript-eslint/consistent-type-assertions - allEvalsResults[idx as number], - { - verbose: opts.showDetailedErrors, - jsonl: opts.jsonl, - }, - ); - - addReport(evalReports, resolvedReporter, report); - } - - if (bundlePromises !== null && Object.entries(evalToExperiment).length > 0) { - await uploadHandleBundles({ - buildResults: buildResults.filter( - // We handle errors above, so it's fine to filter down to successes here. - (result): result is BuildSuccess => result.type === "success", - ), - evalToExperiment, - bundlePromises, - handles, - setCurrent: opts.setCurrent, - defaultIfExists: "replace", - showDetailedErrors: opts.showDetailedErrors, - }); - } - - let allSuccess = true; - for (const [_reporterName, { reporter, results }] of Object.entries( - evalReports, - )) { - const success = await reporter.reportRun(await Promise.all(results)); - allSuccess = allSuccess && success; - } - - return allSuccess; -} - -function checkMatch( - pathInput: string, - include_patterns: string[] | null, - exclude_patterns: string[] | null, -): boolean { - const p = path.resolve(pathInput); - if (include_patterns !== null) { - let include = false; - for (const pattern of include_patterns) { - if (minimatch(p, pattern)) { - include = true; - break; - } - } - if (!include) { - return false; - } - } - - if (exclude_patterns !== null) { - let exclude = false; - for (const pattern of exclude_patterns) { - if (minimatch(p, pattern)) { - exclude = true; - break; - } - } - - return !exclude; - } - - return true; -} - -async function collectFiles( - inputPath: string, - mode: "eval" | "bundle", -): Promise { - let pathStat = null; - try { - pathStat = fs.lstatSync(inputPath); - } catch (e) { - // eslint-disable-next-line no-restricted-properties -- preserving intentional console usage. - console.error(error(`Error reading ${inputPath}: ${e}`)); - process.exit(1); - } - - let files: string[] = []; - if (!pathStat.isDirectory()) { - if ( - !checkMatch( - inputPath, - mode === "eval" ? INCLUDE_EVAL : INCLUDE_BUNDLE, - EXCLUDE, - ) - ) { - const prefix = mode === "eval" ? ".eval" : ""; - // eslint-disable-next-line no-restricted-properties -- preserving intentional console usage. - console.warn( - warning( - `Reading ${inputPath} because it was specified directly. Rename it to end in ${prefix}.ts or ` + - `${prefix}.js to include it automatically when you specify a directory.`, - ), - ); - } - files.push(inputPath); - } else { - const walked = await util.promisify(fsWalk.walk)(inputPath, { - deepFilter: (entry) => { - return checkMatch(entry.path, null, EXCLUDE); - }, - entryFilter: (entry) => { - return ( - entry.dirent.isFile() && - checkMatch( - entry.path, - mode === "eval" ? INCLUDE_EVAL : INCLUDE_BUNDLE, - EXCLUDE, - ) - ); - }, - }); - - files = files.concat(walked.map((entry) => entry.path)); - } - - return files; -} - -import { createMarkKnownPackagesExternalPlugin } from "./util/external-packages-plugin"; - -// Inspired by and modified from https://github.com/evanw/esbuild/issues/1051 -const nativeNodeModulesPlugin = { - name: "native-node-modules", - setup(build: esbuild.PluginBuild) { - // Keep track of packages that contain .node files - const nativePackages = new Set(); - - // Helper to add a package and its platform-specific variants - const addNativePackage = (pkgName: string) => { - nativePackages.add(pkgName); - if (pkgName.includes("@")) { - const [scope, name] = pkgName.split("/"); - const platformPkgs = [ - `${scope}/${name}-darwin-arm64`, - `${scope}/${name}-darwin-x64`, - `${scope}/${name}-linux-x64-gnu`, - `${scope}/${name}-win32-x64-msvc`, - ]; - platformPkgs.forEach((pkg) => nativePackages.add(pkg)); - } - }; - - // When a .node file is imported, mark its package as native - build.onResolve({ filter: /\.node$/ }, (args) => { - try { - const path = require.resolve(args.path, { paths: [args.resolveDir] }); - const match = path.match( - /node_modules[/\\]((?:@[^/\\]+[/\\])?[^/\\]+)/, - ); - if (match) { - addNativePackage(match[1]); - } - } catch { - // Ignore errors - } - return { path: args.path, external: true }; - }); - - // Handle direct imports of native packages - build.onResolve( - { filter: /@[^/]+\/[^/]+-(?:darwin|linux|win32)/ }, - (args) => { - const match = args.path.match(/^(@[^/]+\/[^/]+)/); - if (match) { - addNativePackage(match[1]); - } - return { path: args.path, external: true }; - }, - ); - - // Mark all imports from native packages as external - build.onResolve({ filter: /.*/ }, (args) => { - if (!args.path.startsWith(".") && !args.path.startsWith("/")) { - const match = args.path.match(/^(?:@[^/]+\/)?[^/]+/); - if (match && nativePackages.has(match[0])) { - return { path: require.resolve(args.path), external: true }; - } - } - return null; - }); - }, -}; - -export type PluginMaker = (fileName: string) => esbuild.Plugin; - -function buildOpts({ - fileName, - outFile, - tsconfig, - plugins: argPlugins, - externalPackages, -}: { - fileName: string; - outFile: string; - tsconfig?: string; - plugins?: PluginMaker[]; - externalPackages?: string[]; -}): esbuild.BuildOptions { - const plugins = [ - braintrustEsbuildPlugin(), - nativeNodeModulesPlugin, - createMarkKnownPackagesExternalPlugin(externalPackages), - ...(argPlugins || []).map((fn) => fn(fileName)), - ]; - return { - entryPoints: [fileName], - bundle: true, - treeShaking: true, - outfile: outFile, - platform: "node", - write: false, - // Remove the leading "v" from process.version - target: `node${process.version.slice(1)}`, - tsconfig, - external: ["node_modules/*", "fsevents"], - plugins: plugins, - }; -} - -export async function initializeHandles({ - files: inputFiles, - mode, - plugins, - tsconfig, - externalPackages, -}: { - files: string[]; - mode: "eval" | "bundle"; - plugins?: PluginMaker[]; - tsconfig?: string; - externalPackages?: string[]; -}): Promise> { - const files: Record = {}; - const inputPaths = inputFiles.length > 0 ? inputFiles : ["."]; - for (const inputPath of inputPaths) { - const newFiles = await collectFiles(inputPath, mode); - if (newFiles.length == 0) { - // eslint-disable-next-line no-restricted-properties -- preserving intentional console usage. - console.warn( - warning( - `Provided path ${inputPath} is not an eval file or a directory containing eval files, skipping...`, - ), - ); - } - for (const file of newFiles) { - files[path.resolve(file)] = true; - } - } - - if (Object.keys(files).length == 0) { - // eslint-disable-next-line no-restricted-properties -- preserving intentional console usage. - console.warn( - warning("No eval files were found in any of the provided paths."), - ); - process.exit(0); - } - - const tmpDir = path.join(os.tmpdir(), `btevals-${uuidv4().slice(0, 8)}`); - // fs.mkdirSync(tmpDir, { recursive: true }); - - const initPromises = []; - for (const file of Object.keys(files)) { - const baseName = `${path.basename( - file, - path.extname(file), - )}-${uuidv4().slice(0, 8)}`; - const outFile = path.join(tmpDir, `${baseName}.${OUT_EXT}`); - const bundleFile = path.join(tmpDir, `${baseName}.bundle.js`); - initPromises.push( - initFile({ - inFile: file, - outFile, - bundleFile, - plugins, - tsconfig, - externalPackages, - }), - ); - } - - const handles: Record = {}; - const initResults = await Promise.all(initPromises); - for (const result of initResults) { - handles[result.inFile] = result; - } - return handles; -} - -async function run(args: RunArgs) { - normalizeDebugLoggingArgs(args); - // Load the environment variables from the .env files using the same rules as Next.js - loadEnvConfig(process.cwd(), true); - - if (args.env_file) { - // Load via dotenv library - const loaded = dotenv.config({ path: args.env_file }); - if (loaded.error) { - // eslint-disable-next-line no-restricted-properties -- preserving intentional console usage. - console.error(error(`Error loading ${args.env_file}: ${loaded.error}`)); - process.exit(1); - } - } - - const evaluatorOpts: EvaluatorOpts = { - showDetailedErrors: shouldShowDetailedErrors(args.debug_logging), - apiKey: args.api_key, - orgName: args.org_name, - appUrl: args.app_url, - noSendLogs: !!args.no_send_logs, - bundle: !!args.bundle || !!args.push, - setCurrent: !!args.push, - terminateOnFailure: !!args.terminate_on_failure, - watch: !!args.watch, - jsonl: args.jsonl, - progressReporter: args.no_progress_bars - ? new SimpleProgressReporter() - : new BarProgressReporter(), - filters: args.filter ? parseFilters(args.filter) : [], - list: !!args.list, - }; - - if (args.list && args.watch) { - // eslint-disable-next-line no-restricted-properties -- preserving intentional console usage. - console.error(error("Cannot specify both --list and --watch.")); - process.exit(1); - } - - const plugins = evaluatorOpts.watch - ? [ - (fileName: string) => - buildWatchPluginForEvaluator(fileName, evaluatorOpts), - ] - : []; - - const handles = await initializeHandles({ - files: args.files, - mode: "eval", - tsconfig: args.tsconfig, - plugins, - externalPackages: args.external_packages, - }); - - if (args.dev) { - // XXX We should watch these files (or support a --watch flag). - const { evaluators } = await buildEvaluators(handles, evaluatorOpts); - const allEvaluators = Object.values(evaluators.evaluators).map( - (e) => e.evaluator, - ); - runDevServer(allEvaluators, { - host: args.dev_host, - port: args.dev_port, - orgName: args.dev_org_name, - }); - return; - } - - let success = true; - try { - if (!evaluatorOpts.noSendLogs) { - await login({ - apiKey: args.api_key, - orgName: args.org_name, - appUrl: args.app_url, - debugLogLevel: args.debug_logging, - }); - } - - if (args.watch) { - await runAndWatch({ - handles, - onExit: () => { - evaluatorOpts.progressReporter.stop(); - }, - }); - } else { - success = await runOnce(handles, evaluatorOpts); - } - } finally { - // ESBuild can freeze up if you do not clean up the handles properly - for (const handle of Object.values(handles)) { - await handle.destroy(); - } - } - - if (!success) { - process.exit(1); - } -} - -function addAuthArgs(parser: ArgumentParser) { - parser.add_argument("--api-key", { - help: "Specify a braintrust api key. If the parameter is not specified, BRAINTRUST_API_KEY or the nearest .env.braintrust file will be used.", - }); - parser.add_argument("--org-name", { - help: "The name of a specific organization to connect to. This is useful if you belong to multiple.", - }); - parser.add_argument("--app-url", { - help: "Specify a custom braintrust app url. Defaults to https://www.braintrust.dev. This is only necessary if you are using an experimental version of Braintrust", - }); - parser.add_argument("--env-file", { - help: "A path to a .env file containing environment variables to load (via dotenv).", - }); -} - -function addDebugLoggingArg(parser: ArgumentParser) { - parser.add_argument("--debug-logging", { - choices: ["error", "warn", "info", "debug"], - help: "Enable internal Braintrust SDK troubleshooting output. Use 'error', 'warn', 'info', or 'debug' to control the log level.", - }); -} - -function addCompileArgs(parser: ArgumentParser) { - parser.add_argument("--terminate-on-failure", { - action: "store_true", - help: "If provided, terminates on a failing eval, instead of the default (moving onto the next one).", - }); - parser.add_argument("--tsconfig", { - help: "Specify a custom tsconfig.json file to use.", - }); - parser.add_argument("--external-packages", { - nargs: "*", - help: "Additional packages to mark as external during bundling. These packages will not be included in the bundle and must be available at runtime. Use this to resolve bundling errors with native modules or problematic dependencies. Example: --external-packages sqlite3 fsevents @mapbox/node-pre-gyp", - }); -} - -async function main() { - const parser = new ArgumentParser({ - description: "Braintrust CLI", - }); - - parser.add_argument("-v", "--version", { action: "version", version }); - - const parentParser = new ArgumentParser({ add_help: false }); - parentParser.add_argument("--verbose", { - action: "store_true", - help: "Deprecated alias for --debug-logging debug. Use --debug-logging debug to include full stack traces and detailed troubleshooting output.", - }); - - const subparser = parser.add_subparsers({ - required: true, - }); - - const parser_run = subparser.add_parser("eval", { - help: "Run evals locally.", - parents: [parentParser], - }); - addAuthArgs(parser_run); - addDebugLoggingArg(parser_run); - parser_run.add_argument("--filter", { - help: "Only run evaluators that match these filters. Each filter is a regular expression (https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/RegExp). For example, --filter metadata.priority='^P0$' input.name='foo.*bar' will only run evaluators that have metadata.priority equal to 'P0' and input.name matching the regular expression 'foo.*bar'.", - nargs: "*", - }); - parser_run.add_argument("--list", { - help: "List, but do not execute, evaluators.", - action: "store_true", - }); - parser_run.add_argument("--jsonl", { - action: "store_true", - help: "Format score summaries as jsonl, i.e. one JSON-formatted line per summary.", - }); - addCompileArgs(parser_run); - parser_run.add_argument("--watch", { - action: "store_true", - help: "Watch files for changes and rerun evals when changes are detected", - }); - parser_run.add_argument("--no-send-logs", { - action: "store_true", - help: "Do not send logs to Braintrust. Useful for testing evaluators without uploading results.", - }); - parser_run.add_argument("--no-progress-bars", { - action: "store_true", - help: "Do not show progress bars when processing evaluators.", - }); - parser_run.add_argument("--bundle", { - action: "store_true", - help: "Experimental (do not use unless you know what you're doing)", - }); - parser_run.add_argument("--push", { - action: "store_true", - help: "Push the scorers from the current run to Braintrust. This will mark the current run's scorers as the latest version in the project.", - }); - parser_run.add_argument("files", { - nargs: "*", - help: "A list of files or directories to run. If no files are specified, the current directory is used.", - }); - parser_run.add_argument("--dev", { - action: "store_true", - help: "Run the evaluators in dev mode. This will start a dev server which you can connect to via the playground's remote evals feature.", - }); - parser_run.add_argument("--dev-host", { - help: "The host to bind the dev server to. Defaults to localhost. Set to 0.0.0.0 to bind to all interfaces.", - type: String, - default: "localhost", - }); - parser_run.add_argument("--dev-port", { - help: "The port to bind the dev server to. Defaults to 8300.", - type: Number, - default: 8300, - }); - parser_run.add_argument("--dev-org-name", { - help: "Only allow users that belong this to this org name to run remote evals.", - type: String, - }); - parser_run.set_defaults({ func: run }); - - const parser_push = subparser.add_parser("push", { - help: "Bundle prompts, tools, scorers, and other resources into Braintrust", - }); - addAuthArgs(parser_push); - addDebugLoggingArg(parser_push); - addCompileArgs(parser_push); - parser_push.add_argument("files", { - nargs: "*", - help: "A list of files or directories containing functions to bundle. If no files are specified, the current directory is used.", - }); - parser_push.add_argument("--if-exists", { - choices: ["error", "replace", "ignore"], - default: "error", - help: "What to do if a function with the same slug already exists. 'error' will cause an error and abort. 'replace' will overwrite the existing function. 'ignore' will ignore the push for this function and continue.", - }); - parser_push.set_defaults({ func: bundleCommand }); - - const parser_pull = subparser.add_parser("pull", { - help: "Pull prompts, tools, scorers, and other resources from Braintrust to save in your codebase.", - }); - addDebugLoggingArg(parser_pull); - parser_pull.add_argument("--output-dir", { - help: "The directory to output the pulled resources to. If not specified, the current directory is used.", - }); - parser_pull.add_argument("--project-name", { - help: "The name of the project to pull from. If not specified, all projects are pulled.", - }); - parser_pull.add_argument("--project-id", { - help: "The id of the project to pull from. If not specified, all projects are pulled.", - }); - parser_pull.add_argument("--id", { - help: "The id of a specific function to pull.", - }); - parser_pull.add_argument("--slug", { - help: "The slug of a specific function to pull.", - }); - parser_pull.add_argument("--version", { - help: "The version to pull. Will pull the latest version of each prompt that is at or before this version.", - }); - parser_pull.add_argument("--force", { - action: "store_true", - help: "Overwrite local files if they have uncommitted changes.", - }); - parser_pull.set_defaults({ func: pullCommand }); - - const parsed = normalizeDebugLoggingArgs(parser.parse_args()); - - try { - await parsed.func(parsed); - } catch (e) { - logError(e, shouldShowDetailedErrors(parsed.debug_logging)); - process.exit(1); - } -} - -if (require.main === module) { - void main(); -} diff --git a/js/src/cli/integration.test.ts b/js/src/cli/integration.test.ts deleted file mode 100644 index ae20ec41f..000000000 --- a/js/src/cli/integration.test.ts +++ /dev/null @@ -1,285 +0,0 @@ -import { describe, it, expect } from "vitest"; -import { CompileArgs, RunArgs, BundleArgs, PullArgs } from "./util/types"; - -// Test the type definitions for external packages -describe("External Packages Type Definitions", () => { - describe("CompileArgs interface", () => { - it("should have external_packages property", () => { - const args: CompileArgs = { - tsconfig: "tsconfig.json", - terminate_on_failure: false, - external_packages: ["sqlite3", "fsevents"], - }; - - expect(args.external_packages).toEqual(["sqlite3", "fsevents"]); - }); - - it("should allow undefined external_packages", () => { - const args: CompileArgs = { - tsconfig: "tsconfig.json", - terminate_on_failure: false, - external_packages: undefined, - }; - - expect(args.external_packages).toBeUndefined(); - }); - - it("should allow empty external_packages array", () => { - const args: CompileArgs = { - tsconfig: "tsconfig.json", - terminate_on_failure: false, - external_packages: [], - }; - - expect(args.external_packages).toEqual([]); - }); - }); - - describe("RunArgs interface", () => { - it("should inherit external_packages from CompileArgs", () => { - const args: RunArgs = { - verbose: false, - api_key: "test-key", - org_name: "test-org", - app_url: "https://test.com", - env_file: ".env", - debug_logging: "info", - tsconfig: "tsconfig.json", - terminate_on_failure: false, - external_packages: ["@mapbox/node-pre-gyp"], - files: ["test.eval.ts"], - watch: false, - list: false, - jsonl: false, - filter: undefined, - no_send_logs: false, - no_progress_bars: false, - bundle: false, - push: false, - dev: false, - dev_host: "localhost", - dev_port: 8300, - }; - - expect(args.external_packages).toEqual(["@mapbox/node-pre-gyp"]); - expect(args.debug_logging).toBe("info"); - }); - }); - - describe("BundleArgs interface", () => { - it("should inherit external_packages from CompileArgs", () => { - const args: BundleArgs = { - verbose: false, - api_key: "test-key", - org_name: "test-org", - app_url: "https://test.com", - env_file: ".env", - debug_logging: "debug", - tsconfig: "tsconfig.json", - terminate_on_failure: false, - external_packages: ["sqlite3", "sharp"], - files: ["functions.ts"], - if_exists: "error", - }; - - expect(args.external_packages).toEqual(["sqlite3", "sharp"]); - expect(args.debug_logging).toBe("debug"); - }); - }); - - describe("PullArgs interface", () => { - it("should include debug_logging from CommonArgs", () => { - const args: PullArgs = { - verbose: false, - output_dir: "./braintrust", - project_name: "test-project", - force: false, - debug_logging: "info", - }; - - expect(args.debug_logging).toBe("info"); - }); - }); -}); - -// Test the buildOpts function parameter flow -describe("buildOpts Function Integration", () => { - // This is a simplified version of the buildOpts function signature - interface BuildOptsParams { - fileName: string; - outFile: string; - tsconfig?: string; - plugins?: any[]; - externalPackages?: string[]; - } - - it("should accept externalPackages parameter", () => { - const params: BuildOptsParams = { - fileName: "test.ts", - outFile: "test.js", - tsconfig: "tsconfig.json", - plugins: [], - externalPackages: ["sqlite3", "fsevents"], - }; - - expect(params.externalPackages).toEqual(["sqlite3", "fsevents"]); - }); - - it("should handle undefined externalPackages", () => { - const params: BuildOptsParams = { - fileName: "test.ts", - outFile: "test.js", - tsconfig: "tsconfig.json", - plugins: [], - externalPackages: undefined, - }; - - expect(params.externalPackages).toBeUndefined(); - }); - - it("should handle empty externalPackages array", () => { - const params: BuildOptsParams = { - fileName: "test.ts", - outFile: "test.js", - tsconfig: "tsconfig.json", - plugins: [], - externalPackages: [], - }; - - expect(params.externalPackages).toEqual([]); - }); -}); - -// Test the parameter flow from CLI to buildOpts -describe("Parameter Flow Integration", () => { - it("should flow external_packages from RunArgs to buildOpts", () => { - // Simulate the parameter flow: RunArgs -> initializeHandles -> initFile -> buildOpts - const runArgs: Partial = { - external_packages: ["sqlite3", "@mapbox/node-pre-gyp"], - }; - - // This would be passed to initializeHandles - const initializeHandlesParams = { - files: ["test.eval.ts"], - mode: "eval" as const, - tsconfig: undefined, - plugins: undefined, - externalPackages: runArgs.external_packages, - }; - - expect(initializeHandlesParams.externalPackages).toEqual([ - "sqlite3", - "@mapbox/node-pre-gyp", - ]); - - // This would be passed to initFile - const initFileParams = { - inFile: "test.eval.ts", - outFile: "test.js", - bundleFile: "test.bundle.js", - tsconfig: undefined, - plugins: undefined, - externalPackages: initializeHandlesParams.externalPackages, - }; - - expect(initFileParams.externalPackages).toEqual([ - "sqlite3", - "@mapbox/node-pre-gyp", - ]); - - // This would be passed to buildOpts - const buildOptsParams = { - fileName: initFileParams.inFile, - outFile: initFileParams.outFile, - tsconfig: initFileParams.tsconfig, - plugins: initFileParams.plugins, - externalPackages: initFileParams.externalPackages, - }; - - expect(buildOptsParams.externalPackages).toEqual([ - "sqlite3", - "@mapbox/node-pre-gyp", - ]); - }); - - it("should flow external_packages from BundleArgs to buildOpts", () => { - // Simulate the parameter flow: BundleArgs -> initializeHandles -> initFile -> buildOpts - const bundleArgs: Partial = { - external_packages: ["sharp", "canvas"], - }; - - // This would be passed to initializeHandles - const initializeHandlesParams = { - files: ["functions.ts"], - mode: "bundle" as const, - tsconfig: undefined, - plugins: undefined, - externalPackages: bundleArgs.external_packages, - }; - - expect(initializeHandlesParams.externalPackages).toEqual([ - "sharp", - "canvas", - ]); - - // This would be passed to buildOpts - const buildOptsParams = { - fileName: "functions.ts", - outFile: "functions.js", - tsconfig: undefined, - plugins: undefined, - externalPackages: initializeHandlesParams.externalPackages, - }; - - expect(buildOptsParams.externalPackages).toEqual(["sharp", "canvas"]); - }); -}); - -// Test real-world usage scenarios -describe("Real-world Usage Scenarios", () => { - it("should handle autoevals dependency issue scenario", () => { - const args: Partial = { - files: ["autoevals-test.eval.ts"], - external_packages: ["@mapbox/node-pre-gyp", "mock-aws-s3", "aws-sdk"], - }; - - // These packages would be added to the external list - const expectedExternals = [ - // Hardcoded externals - "braintrust", - "autoevals", - "@braintrust/", - "config", - "lightningcss", - "@mapbox/node-pre-gyp", - // CLI-specified externals - ...args.external_packages!, - ]; - - expect(expectedExternals).toContain("@mapbox/node-pre-gyp"); - expect(expectedExternals).toContain("mock-aws-s3"); - expect(expectedExternals).toContain("aws-sdk"); - }); - - it("should handle database packages scenario", () => { - const args: Partial = { - files: ["database-test.eval.ts"], - external_packages: ["sqlite3", "better-sqlite3", "mysql2"], - }; - - expect(args.external_packages).toEqual([ - "sqlite3", - "better-sqlite3", - "mysql2", - ]); - }); - - it("should handle native modules scenario", () => { - const args: Partial = { - files: ["image-processing.eval.ts"], - external_packages: ["sharp", "canvas", "node-sass"], - }; - - expect(args.external_packages).toEqual(["sharp", "canvas", "node-sass"]); - }); -}); diff --git a/js/src/cli/jest/nodeModulesPaths.ts b/js/src/cli/jest/nodeModulesPaths.ts deleted file mode 100644 index f04748e9a..000000000 --- a/js/src/cli/jest/nodeModulesPaths.ts +++ /dev/null @@ -1,74 +0,0 @@ -/** - * Copyright (c) Meta Platforms, Inc. and affiliates. - * - * This source code is licensed under the MIT license found in the - * LICENSE file in the root directory of this source tree. - * - * Adapted from: https://github.com/substack/node-resolve - */ - -import * as path from "node:path"; -import { realpathSync } from "node:fs"; - -type NodeModulesPathsOptions = { - moduleDirectory?: Array; - paths?: Array; -}; - -export default function nodeModulesPaths( - basedir: string, - options: NodeModulesPathsOptions, -): Array { - const modules = - options && options.moduleDirectory - ? Array.from(options.moduleDirectory) - : ["node_modules"]; - - // ensure that `basedir` is an absolute path at this point, - // resolving against the process' current working directory - const basedirAbs = path.resolve(basedir); - - let prefix = "/"; - if (/^([A-Za-z]:)/.test(basedirAbs)) { - prefix = ""; - } else if (/^\\\\/.test(basedirAbs)) { - prefix = "\\\\"; - } - - // The node resolution algorithm (as implemented by NodeJS and TypeScript) - // traverses parents of the physical path, not the symlinked path - let physicalBasedir; - try { - physicalBasedir = realpathSync.native(basedirAbs); - } catch { - // realpath can throw, e.g. on mapped drives - physicalBasedir = basedirAbs; - } - - const paths: Array = [physicalBasedir]; - let parsed = path.parse(physicalBasedir); - while (parsed.dir !== paths[paths.length - 1]) { - paths.push(parsed.dir); - parsed = path.parse(parsed.dir); - } - - const dirs = paths.reduce>((dirs, aPath) => { - for (const moduleDir of modules) { - if (path.isAbsolute(moduleDir)) { - if (aPath === basedirAbs && moduleDir) { - dirs.push(moduleDir); - } - } else { - dirs.push(path.join(prefix, aPath, moduleDir)); - } - } - - return dirs; - }, []); - - if (options.paths) { - dirs.push(...options.paths); - } - - return dirs; -} diff --git a/js/src/cli/progress.ts b/js/src/cli/progress.ts deleted file mode 100644 index befab92f6..000000000 --- a/js/src/cli/progress.ts +++ /dev/null @@ -1 +0,0 @@ -export { BarProgressReporter } from "./reporters/progress"; diff --git a/js/src/cli/reporters/eval.ts b/js/src/cli/reporters/eval.ts deleted file mode 100644 index 8f7ff8708..000000000 --- a/js/src/cli/reporters/eval.ts +++ /dev/null @@ -1,227 +0,0 @@ -import { stripVTControlCharacters, styleText } from "node:util"; -import { terminalLink } from "termi-link"; -import Table from "cli-table3"; -import pluralize from "pluralize"; - -import { ExperimentSummary, ScoreSummary, MetricSummary } from "../../logger"; -import type { ReporterDef } from "../../reporters/types"; -import { EvaluatorDef, EvalResultWithSummary } from "../../framework"; -import { isEmpty } from "../../util"; - -function visibleLength(text: string) { - return stripVTControlCharacters(text).length; -} - -function padEndVisible(text: string, targetLength: number) { - return text + " ".repeat(Math.max(0, targetLength - visibleLength(text))); -} - -function formatSummaryBox(content: string) { - const title = styleText("gray", " Experiment summary "); - const lines = content.split("\n"); - const contentWidth = Math.max( - visibleLength(title), - ...lines.map((line) => visibleLength(line) + 2), - ); - - const horizontal = "─"; - const top = - styleText("gray", "╭") + - title + - styleText( - "gray", - horizontal.repeat(contentWidth - visibleLength(title)) + "╮", - ); - const body = lines - .map( - (line) => - `${styleText("gray", "│")} ${padEndVisible(line, contentWidth - 2)} ${styleText("gray", "│")}`, - ) - .join("\n"); - const bottom = styleText("gray", "╰" + horizontal.repeat(contentWidth) + "╯"); - - return top + "\n" + body + "\n" + bottom; -} - -function formatExperimentSummaryFancy(summary: ExperimentSummary) { - let comparisonLine = ""; - if (summary.comparisonExperimentName) { - comparisonLine = `${summary.comparisonExperimentName} ${styleText("gray", "(baseline)")} ← ${summary.experimentName} ${styleText("gray", "(comparison)")}\n\n`; - } - - const tableParts: string[] = []; - - const hasScores = Object.keys(summary.scores).length > 0; - const hasMetrics = Object.keys(summary.metrics ?? {}).length > 0; - const hasComparison = !!summary.comparisonExperimentName; - - if (hasScores || hasMetrics) { - const headers = [styleText("gray", "Name"), styleText("gray", "Value")]; - - if (hasComparison) { - headers.push( - styleText("gray", "Change"), - styleText("gray", "Improvements"), - styleText("gray", "Regressions"), - ); - } - - const combinedTable = new Table({ - head: hasComparison ? headers : [], - style: { head: [], "padding-left": 0, "padding-right": 0, border: [] }, - chars: { - top: "", - "top-mid": "", - "top-left": "", - "top-right": "", - bottom: "", - "bottom-mid": "", - "bottom-left": "", - "bottom-right": "", - left: "", - "left-mid": "", - mid: "", - "mid-mid": "", - right: "", - "right-mid": "", - middle: " ", - }, - colWidths: hasComparison ? [18, 10, 10, 13, 12] : [20, 15], - colAligns: hasComparison - ? ["left", "right", "right", "right", "right"] - : ["left", "right"], - wordWrap: false, - }); - - const scoreValues: ScoreSummary[] = Object.values(summary.scores); - for (const score of scoreValues) { - const scorePercent = (score.score * 100).toFixed(2); - const scoreValue = styleText("white", `${scorePercent}%`); - - let diffString = ""; - if (!isEmpty(score.diff)) { - const diffPercent = (score.diff! * 100).toFixed(2); - const diffSign = score.diff! > 0 ? "+" : ""; - const diffColor = score.diff! > 0 ? "green" : "red"; - diffString = styleText(diffColor, `${diffSign}${diffPercent}%`); - } else { - diffString = styleText("gray", "-"); - } - - const improvements = - score.improvements > 0 - ? styleText(["dim", "green"], String(score.improvements)) - : styleText("gray", "-"); - const regressions = - score.regressions > 0 - ? styleText(["dim", "red"], String(score.regressions)) - : styleText("gray", "-"); - - const row = [`${styleText("blue", "◯")} ${score.name}`, scoreValue]; - if (hasComparison) { - row.push(diffString, improvements, regressions); - } - combinedTable.push(row); - } - - const metricValues: MetricSummary[] = Object.values(summary.metrics ?? {}); - for (const metric of metricValues) { - const fractionDigits = Number.isInteger(metric.metric) ? 0 : 2; - const formattedValue = metric.metric.toFixed(fractionDigits); - const metricValue = styleText( - "white", - metric.unit === "$" - ? `${metric.unit}${formattedValue}` - : `${formattedValue}${metric.unit}`, - ); - - let diffString = ""; - if (!isEmpty(metric.diff)) { - const diffPercent = (metric.diff! * 100).toFixed(2); - const diffSign = metric.diff! > 0 ? "+" : ""; - const diffColor = metric.diff! > 0 ? "green" : "red"; - diffString = styleText(diffColor, `${diffSign}${diffPercent}%`); - } else { - diffString = styleText("gray", "-"); - } - - const improvements = - metric.improvements > 0 - ? styleText(["dim", "green"], String(metric.improvements)) - : styleText("gray", "-"); - const regressions = - metric.regressions > 0 - ? styleText(["dim", "red"], String(metric.regressions)) - : styleText("gray", "-"); - - const row = [`${styleText("magenta", "◯")} ${metric.name}`, metricValue]; - if (hasComparison) { - row.push(diffString, improvements, regressions); - } - combinedTable.push(row); - } - - tableParts.push(combinedTable.toString()); - } - - const content = [comparisonLine, ...tableParts].filter(Boolean).join("\n"); - - const footer = summary.experimentUrl - ? terminalLink( - `View results for ${summary.experimentName}`, - summary.experimentUrl, - { fallback: () => `See results at ${summary.experimentUrl}` }, - ) - : ""; - - const boxContent = [content, footer].filter(Boolean).join("\n\n"); - - return "\n" + formatSummaryBox(boxContent); -} - -export const warning = (text: string) => styleText("yellow", text); - -export const fancyReporter: ReporterDef = { - name: "Braintrust fancy reporter", - async reportEval( - // eslint-disable-next-line @typescript-eslint/no-explicit-any - evaluator: EvaluatorDef, - // eslint-disable-next-line @typescript-eslint/no-explicit-any - result: EvalResultWithSummary, - { verbose, jsonl }: { verbose: boolean; jsonl?: boolean }, - ) { - const { results, summary } = result; - const failingResults = results.filter( - (r: { error: unknown }) => r.error !== undefined, - ); - - if (failingResults.length > 0) { - // eslint-disable-next-line no-restricted-properties -- preserving intentional console usage. - console.error( - warning( - `Evaluator ${evaluator.evalName} failed with ${pluralize("error", failingResults.length, true)}. This evaluation ("${evaluator.evalName}") will not be fully logged.`, - ), - ); - if (jsonl) { - for (const result of failingResults) { - process.stdout.write(JSON.stringify(result)); - process.stdout.write("\n"); - } - } else if (verbose) { - for (const result of failingResults) { - // eslint-disable-next-line no-restricted-properties -- preserving intentional console usage. - console.error(result); - } - } - } - - process.stdout.write( - jsonl ? JSON.stringify(summary) : formatExperimentSummaryFancy(summary), - ); - process.stdout.write("\n"); - return failingResults.length === 0; - }, - async reportRun(evalReports: boolean[]) { - return evalReports.every((r) => r); - }, -}; diff --git a/js/src/cli/reporters/progress.ts b/js/src/cli/reporters/progress.ts deleted file mode 100644 index e764709f3..000000000 --- a/js/src/cli/reporters/progress.ts +++ /dev/null @@ -1,51 +0,0 @@ -import { styleText } from "node:util"; -import * as cliProgress from "cli-progress"; - -import type { ProgressReporter } from "../../reporters/types"; - -export { SimpleProgressReporter } from "../../reporters/progress"; - -const MAX_NAME_LENGTH = 40; - -function fitNameToSpaces(name: string, length: number) { - const padded = name.padEnd(length); - if (padded.length <= length) { - return padded; - } - return padded.substring(0, length - 3) + "..."; -} - -export class BarProgressReporter implements ProgressReporter { - private multiBar: cliProgress.MultiBar; - private bars: Record = {}; - - constructor() { - this.multiBar = new cliProgress.MultiBar( - { - format: `${styleText("blueBright", "{bar}")} ${styleText("blue", "{evaluator}")} {percentage}% ${styleText("gray", "{value}/{total} {eta_formatted}")}`, - hideCursor: true, - barsize: 10, - }, - cliProgress.Presets.shades_grey, - ); - } - - public start(name: string, total: number) { - const bar = this.multiBar.create(total, 0); - this.bars[name] = bar; - } - - public stop() { - this.multiBar.stop(); - } - - public increment(name: string) { - this.bars[name].increment({ - evaluator: fitNameToSpaces(name, MAX_NAME_LENGTH), - }); - } - - public setTotal(name: string, total: number) { - this.bars[name].setTotal(total); - } -} diff --git a/js/src/cli/types.ts b/js/src/cli/types.ts deleted file mode 100644 index 15184a247..000000000 --- a/js/src/cli/types.ts +++ /dev/null @@ -1,40 +0,0 @@ -import type * as esbuild from "esbuild"; -import type { BaseMetadata } from "../logger"; -import type { EvaluatorDef, EvaluatorFile } from "../framework"; -import type { ReporterDef } from "../reporters/types"; - -export interface BuildSuccess { - type: "success"; - result: esbuild.BuildResult; - evaluator: EvaluatorFile; - sourceFile: string; -} - -export interface BuildFailure { - type: "failure"; - error: Error; - sourceFile: string; -} - -export type BtBuildResult = BuildSuccess | BuildFailure; - -export interface FileHandle { - inFile: string; - outFile: string; - bundleFile?: string; - rebuild: () => Promise; - bundle: () => Promise; - watch: () => void; - destroy: () => Promise; -} - -export interface EvaluatorState { - evaluators: { - sourceFile: string; - evaluator: EvaluatorDef; - reporter: string | ReporterDef | undefined; - }[]; - reporters: { - [reporter: string]: ReporterDef; - }; -} diff --git a/js/src/cli/util/bundle.ts b/js/src/cli/util/bundle.ts deleted file mode 100644 index 9eb18bced..000000000 --- a/js/src/cli/util/bundle.ts +++ /dev/null @@ -1,91 +0,0 @@ -import { loadEnvConfig } from "@next/env"; -import * as dotenv from "dotenv"; -import { AuthArgs, BundleArgs, CommonArgs } from "./types"; -import { error } from "../../framework"; -import type { BtBuildResult } from "../types"; -import { handleBuildFailure, initializeHandles } from "../index"; -import { login } from "../../logger"; -import { uploadHandleBundles } from "../functions/upload"; -import { - normalizeDebugLoggingArgs, - shouldShowDetailedErrors, -} from "./debug-logging"; - -export async function loadCLIEnv(args: AuthArgs & CommonArgs) { - normalizeDebugLoggingArgs(args); - // Load the environment variables from the .env files using the same rules as Next.js - loadEnvConfig(process.cwd(), true); - - if (args.env_file) { - // Load via dotenv library - const loaded = dotenv.config({ path: args.env_file }); - if (loaded.error) { - // eslint-disable-next-line no-restricted-properties -- preserving intentional console usage. - console.error(error(`Error loading ${args.env_file}: ${loaded.error}`)); - process.exit(1); - } - } - - await login({ - apiKey: args.api_key, - orgName: args.org_name, - appUrl: args.app_url, - debugLogLevel: args.debug_logging, - }); -} - -export async function bundleCommand(args: BundleArgs) { - await loadCLIEnv(args); - const showDetailedErrors = shouldShowDetailedErrors(args.debug_logging); - - const handles = await initializeHandles({ - mode: "bundle", - files: args.files, - tsconfig: args.tsconfig, - externalPackages: args.external_packages, - }); - - try { - const allBuildResultsP: Promise[] = Object.values( - handles, - ).map((handle) => handle.rebuild()); - - const bundlePromises = Object.fromEntries( - Object.entries(handles).map(([inFile, handle]) => [ - inFile, - handle.bundle(), - ]), - ); - - const allBuildResults = await Promise.all(allBuildResultsP); - const buildResults = []; - for (const buildResult of allBuildResults) { - if (buildResult.type === "failure") { - handleBuildFailure({ - result: buildResult, - terminateOnFailure: args.terminate_on_failure, - showDetailedErrors, - }); - } else { - buildResults.push(buildResult); - } - } - - const { numFailed } = await uploadHandleBundles({ - buildResults, - bundlePromises, - handles, - setCurrent: true, - showDetailedErrors, - defaultIfExists: args.if_exists, - }); - - if (numFailed > 0) { - process.exit(1); - } - } finally { - for (const handle of Object.values(handles)) { - await handle.destroy(); - } - } -} diff --git a/js/src/cli/util/debug-logging.test.ts b/js/src/cli/util/debug-logging.test.ts deleted file mode 100644 index 320911e5e..000000000 --- a/js/src/cli/util/debug-logging.test.ts +++ /dev/null @@ -1,53 +0,0 @@ -import { beforeEach, describe, expect, test, vi } from "vitest"; - -import { - normalizeDebugLoggingArgs, - resetDebugLoggingArgsForTests, - shouldShowDetailedErrors, - VERBOSE_DEPRECATION_MESSAGE, -} from "./debug-logging"; - -describe("CLI debug logging helpers", () => { - beforeEach(() => { - vi.restoreAllMocks(); - resetDebugLoggingArgsForTests(); - }); - - test("shouldShowDetailedErrors only enables detailed errors for debug", () => { - expect(shouldShowDetailedErrors(undefined)).toBe(false); - expect(shouldShowDetailedErrors("info")).toBe(false); - expect(shouldShowDetailedErrors("debug")).toBe(true); - }); - - test("normalizeDebugLoggingArgs upgrades verbose to debug and warns once", () => { - const warnSpy = vi.spyOn(console, "warn").mockImplementation(() => {}); - const args = { - verbose: true, - debug_logging: undefined as - | "error" - | "warn" - | "info" - | "debug" - | undefined, - }; - - normalizeDebugLoggingArgs(args); - normalizeDebugLoggingArgs(args); - - expect(args.debug_logging).toBe("debug"); - expect(warnSpy).toHaveBeenCalledTimes(1); - expect(warnSpy).toHaveBeenCalledWith( - `Warning: ${VERBOSE_DEPRECATION_MESSAGE}`, - ); - }); - - test("normalizeDebugLoggingArgs preserves explicit debug logging", () => { - const warnSpy = vi.spyOn(console, "warn").mockImplementation(() => {}); - const args = { verbose: true, debug_logging: "info" as const }; - - normalizeDebugLoggingArgs(args); - - expect(args.debug_logging).toBe("info"); - expect(warnSpy).toHaveBeenCalledTimes(1); - }); -}); diff --git a/js/src/cli/util/debug-logging.ts b/js/src/cli/util/debug-logging.ts deleted file mode 100644 index 544514a11..000000000 --- a/js/src/cli/util/debug-logging.ts +++ /dev/null @@ -1,37 +0,0 @@ -import { warning } from "../../framework"; -import type { CommonArgs } from "./types"; - -export const VERBOSE_DEPRECATION_MESSAGE = - "--verbose is deprecated and will be removed in a future version of braintrust. Use --debug-logging debug to see full stack traces and troubleshooting details."; - -let hasWarnedAboutVerboseFlag = false; - -export function shouldShowDetailedErrors( - debugLogLevel: CommonArgs["debug_logging"] | undefined, -): boolean { - return debugLogLevel === "debug"; -} - -export function normalizeDebugLoggingArgs< - T extends Pick, ->(args: T): T { - if (!args.verbose) { - return args; - } - - if (!hasWarnedAboutVerboseFlag) { - hasWarnedAboutVerboseFlag = true; - // eslint-disable-next-line no-restricted-properties -- preserving intentional console usage. - console.warn(warning(VERBOSE_DEPRECATION_MESSAGE)); - } - - if (!args.debug_logging) { - args.debug_logging = "debug"; - } - - return args; -} - -export function resetDebugLoggingArgsForTests(): void { - hasWarnedAboutVerboseFlag = false; -} diff --git a/js/src/cli/util/external-packages-plugin.test.ts b/js/src/cli/util/external-packages-plugin.test.ts deleted file mode 100644 index 34011d05f..000000000 --- a/js/src/cli/util/external-packages-plugin.test.ts +++ /dev/null @@ -1,319 +0,0 @@ -import { describe, it, expect, vi } from "vitest"; -import { createMarkKnownPackagesExternalPlugin } from "./external-packages-plugin"; - -describe("External Packages Plugin", () => { - describe("createMarkKnownPackagesExternalPlugin", () => { - it("should create a plugin with the correct name", () => { - const plugin = createMarkKnownPackagesExternalPlugin(); - expect(plugin.name).toBe("make-known-packages-external"); - }); - - it("should mark hardcoded packages as external", () => { - const plugin = createMarkKnownPackagesExternalPlugin(); - const mockBuild = { - onResolve: vi.fn(), - }; - - plugin.setup(mockBuild as any); - - expect(mockBuild.onResolve).toHaveBeenCalledWith( - { filter: expect.any(RegExp) }, - expect.any(Function), - ); - - // Get the filter regex that was passed - const [{ filter }] = mockBuild.onResolve.mock.calls[0]; - const resolveFunction = mockBuild.onResolve.mock.calls[0][1]; - - // Test hardcoded packages - expect(filter.test("braintrust")).toBe(true); - expect(filter.test("autoevals")).toBe(true); - expect(filter.test("@mapbox/node-pre-gyp")).toBe(true); - expect(filter.test("config")).toBe(true); - expect(filter.test("lightningcss")).toBe(true); - - // Test that the resolve function returns external: true - expect(resolveFunction({ path: "braintrust" })).toEqual({ - path: "braintrust", - external: true, - }); - }); - - it("should not mark non-matching packages as external", () => { - const plugin = createMarkKnownPackagesExternalPlugin(); - const mockBuild = { - onResolve: vi.fn(), - }; - - plugin.setup(mockBuild as any); - - const [{ filter }] = mockBuild.onResolve.mock.calls[0]; - - // Test packages that should NOT match - expect(filter.test("react")).toBe(false); - expect(filter.test("lodash")).toBe(false); - expect(filter.test("@types/node")).toBe(false); - expect(filter.test("some-random-package")).toBe(false); - }); - - it("should include additional packages from CLI", () => { - const additionalPackages = ["sqlite3", "fsevents", "@scope/package"]; - const plugin = createMarkKnownPackagesExternalPlugin(additionalPackages); - const mockBuild = { - onResolve: vi.fn(), - }; - - plugin.setup(mockBuild as any); - - const [{ filter }] = mockBuild.onResolve.mock.calls[0]; - - // Test that additional packages are included - expect(filter.test("sqlite3")).toBe(true); - expect(filter.test("fsevents")).toBe(true); - expect(filter.test("@scope/package")).toBe(true); - - // Test that hardcoded packages still work - expect(filter.test("braintrust")).toBe(true); - expect(filter.test("@mapbox/node-pre-gyp")).toBe(true); - }); - - it("should handle packages with subpaths", () => { - const plugin = createMarkKnownPackagesExternalPlugin(); - const mockBuild = { - onResolve: vi.fn(), - }; - - plugin.setup(mockBuild as any); - - const [{ filter }] = mockBuild.onResolve.mock.calls[0]; - - // Test subpaths - expect(filter.test("braintrust/core")).toBe(true); - expect(filter.test("braintrust/dist/index.js")).toBe(true); - expect( - filter.test("@mapbox/node-pre-gyp/lib/util/nw-pre-gyp/index.html"), - ).toBe(true); - }); - - it("should handle special characters in package names", () => { - const additionalPackages = [ - "@scope/package-with-dashes", - "package.with.dots", - ]; - const plugin = createMarkKnownPackagesExternalPlugin(additionalPackages); - const mockBuild = { - onResolve: vi.fn(), - }; - - plugin.setup(mockBuild as any); - - const [{ filter }] = mockBuild.onResolve.mock.calls[0]; - - // Test packages with special characters - expect(filter.test("@scope/package-with-dashes")).toBe(true); - expect(filter.test("package.with.dots")).toBe(true); - }); - - it("should resolve the original @mapbox/node-pre-gyp bundling issue", () => { - const plugin = createMarkKnownPackagesExternalPlugin(); - const mockBuild = { - onResolve: vi.fn(), - }; - - plugin.setup(mockBuild as any); - - const [{ filter }] = mockBuild.onResolve.mock.calls[0]; - const resolveFunction = mockBuild.onResolve.mock.calls[0][1]; - - // Test the specific problematic paths from the original issue - const problematicPaths = [ - "@mapbox/node-pre-gyp", - "@mapbox/node-pre-gyp/lib/util/nw-pre-gyp/index.html", - "@mapbox/node-pre-gyp/lib/util/s3_setup.js", - "@mapbox/node-pre-gyp/lib/node-pre-gyp.js", - ]; - - problematicPaths.forEach((path) => { - expect(filter.test(path)).toBe(true); - expect(resolveFunction({ path })).toEqual({ - path, - external: true, - }); - }); - }); - - it("should prevent false positives with similar package names", () => { - const plugin = createMarkKnownPackagesExternalPlugin(); - const mockBuild = { - onResolve: vi.fn(), - }; - - plugin.setup(mockBuild as any); - - const [{ filter }] = mockBuild.onResolve.mock.calls[0]; - - // Test cases that should NOT match (false positives) - const falsePositives = [ - "braintrust-extended", - "my-braintrust", - "config-loader", - "lightningcss-plugin", - "sqlite3-wrapper", - "better-sqlite3", - "node-pre-gyp", // Similar to @mapbox/node-pre-gyp but different - "@other/braintrust", - ]; - - falsePositives.forEach((packageName) => { - expect(filter.test(packageName)).toBe(false); - }); - - // Test cases that SHOULD match (true positives) - const truePositives = [ - "braintrust", - "braintrust/core", - "braintrust/dist/index.js", - "@mapbox/node-pre-gyp", - "@mapbox/node-pre-gyp/lib/util/nw-pre-gyp/index.html", - "config", - "lightningcss", - ]; - - truePositives.forEach((packageName) => { - expect(filter.test(packageName)).toBe(true); - }); - }); - - it("should handle prefix matching vs exact matching correctly", () => { - const plugin = createMarkKnownPackagesExternalPlugin(); - const mockBuild = { - onResolve: vi.fn(), - }; - - plugin.setup(mockBuild as any); - - const [{ filter }] = mockBuild.onResolve.mock.calls[0]; - - // Test prefix matching for @braintrust/ (ends with /) - const braintrustPrefixTests = [ - { - input: "@braintrust/utils", - expected: true, - description: "Should match @braintrust/ prefix", - }, - { - input: "@braintrust-other/core", - expected: false, - description: "Should not match similar but different scope", - }, - ]; - - // Test exact matching for config (doesn't end with /) - const configExactTests = [ - { - input: "config", - expected: true, - description: "Should match config exactly", - }, - { - input: "config/local", - expected: true, - description: "Should match config with subpath", - }, - { - input: "config-loader", - expected: false, - description: "Should not match config with suffix", - }, - { - input: "my-config", - expected: false, - description: "Should not match config with prefix", - }, - ]; - - // Test exact matching for braintrust (doesn't end with /) - const braintrustExactTests = [ - { - input: "braintrust", - expected: true, - description: "Should match braintrust exactly", - }, - { - input: "braintrust/core", - expected: true, - description: "Should match braintrust with subpath", - }, - { - input: "braintrust-extended", - expected: false, - description: "Should not match braintrust with suffix", - }, - { - input: "my-braintrust", - expected: false, - description: "Should not match braintrust with prefix", - }, - ]; - - const allTests = [ - ...braintrustPrefixTests, - ...configExactTests, - ...braintrustExactTests, - ]; - - allTests.forEach((test) => { - const result = filter.test(test.input); - expect(result).toBe(test.expected); - }); - }); - }); - - describe("Regex Edge Cases", () => { - it("should handle empty additional packages array", () => { - const plugin = createMarkKnownPackagesExternalPlugin([]); - const mockBuild = { - onResolve: vi.fn(), - }; - - plugin.setup(mockBuild as any); - - const [{ filter }] = mockBuild.onResolve.mock.calls[0]; - - // Should still work with hardcoded packages - expect(filter.test("braintrust")).toBe(true); - expect(filter.test("@mapbox/node-pre-gyp")).toBe(true); - }); - - it("should handle undefined additional packages", () => { - const plugin = createMarkKnownPackagesExternalPlugin(undefined); - const mockBuild = { - onResolve: vi.fn(), - }; - - plugin.setup(mockBuild as any); - - const [{ filter }] = mockBuild.onResolve.mock.calls[0]; - - // Should still work with hardcoded packages - expect(filter.test("braintrust")).toBe(true); - expect(filter.test("@mapbox/node-pre-gyp")).toBe(true); - }); - - it("should handle duplicate packages", () => { - const additionalPackages = ["braintrust", "sqlite3", "braintrust"]; // duplicate - const plugin = createMarkKnownPackagesExternalPlugin(additionalPackages); - const mockBuild = { - onResolve: vi.fn(), - }; - - plugin.setup(mockBuild as any); - - const [{ filter }] = mockBuild.onResolve.mock.calls[0]; - - // Should still work correctly - expect(filter.test("braintrust")).toBe(true); - expect(filter.test("sqlite3")).toBe(true); - }); - }); -}); diff --git a/js/src/cli/util/external-packages-plugin.ts b/js/src/cli/util/external-packages-plugin.ts deleted file mode 100644 index 115a98bff..000000000 --- a/js/src/cli/util/external-packages-plugin.ts +++ /dev/null @@ -1,43 +0,0 @@ -import * as esbuild from "esbuild"; - -// Inspired by https://github.com/evanw/esbuild/issues/619 -// In addition to marking node_modules external, explicitly mark -// our packages (braintrust and autoevals) external, in case they're -// installed in a relative path. -export function createMarkKnownPackagesExternalPlugin( - additionalPackages: string[] = [], -) { - return { - name: "make-known-packages-external", - setup(build: esbuild.PluginBuild) { - // Mark known packages as external - const knownPackages = [ - "braintrust", - "autoevals", - "@braintrust/", - "config", - "lightningcss", - "@mapbox/node-pre-gyp", - "fsevents", - "chokidar", - ...additionalPackages, - ]; - const escapedPackages = knownPackages.map((pkg) => { - const escaped = pkg.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); - // For packages ending with /, match anything with that prefix - if (pkg.endsWith("/")) { - return escaped + ".*"; - } - // For regular packages, match exact name or name followed by / - return escaped + "(?:\\/.*)?"; - }); - const knownPackagesFilter = new RegExp( - `^(${escapedPackages.join("|")})$`, - ); - build.onResolve({ filter: knownPackagesFilter }, (args) => ({ - path: args.path, - external: true, - })); - }, - }; -} diff --git a/js/src/cli/util/external-packages.test.ts b/js/src/cli/util/external-packages.test.ts deleted file mode 100644 index 210279917..000000000 --- a/js/src/cli/util/external-packages.test.ts +++ /dev/null @@ -1,199 +0,0 @@ -import { describe, it, expect } from "vitest"; -import { ArgumentParser } from "argparse"; - -// Test the CLI argument parsing for external packages -describe("External Packages CLI Arguments", () => { - describe("--external-packages flag", () => { - it("should parse single package", () => { - const parser = new ArgumentParser(); - parser.add_argument("--external-packages", { - nargs: "*", - help: "Additional packages to mark as external during bundling.", - }); - - const args = parser.parse_args(["--external-packages", "sqlite3"]); - expect(args.external_packages).toEqual(["sqlite3"]); - }); - - it("should parse multiple packages", () => { - const parser = new ArgumentParser(); - parser.add_argument("--external-packages", { - nargs: "*", - help: "Additional packages to mark as external during bundling.", - }); - - const args = parser.parse_args([ - "--external-packages", - "sqlite3", - "fsevents", - "@mapbox/node-pre-gyp", - ]); - expect(args.external_packages).toEqual([ - "sqlite3", - "fsevents", - "@mapbox/node-pre-gyp", - ]); - }); - - it("should handle scoped packages", () => { - const parser = new ArgumentParser(); - parser.add_argument("--external-packages", { - nargs: "*", - help: "Additional packages to mark as external during bundling.", - }); - - const args = parser.parse_args([ - "--external-packages", - "@scope/package", - "@another/scoped-package", - ]); - expect(args.external_packages).toEqual([ - "@scope/package", - "@another/scoped-package", - ]); - }); - - it("should handle packages with special characters", () => { - const parser = new ArgumentParser(); - parser.add_argument("--external-packages", { - nargs: "*", - help: "Additional packages to mark as external during bundling.", - }); - - const args = parser.parse_args([ - "--external-packages", - "package-with-dashes", - "package.with.dots", - "@scope/package-with-dashes", - ]); - expect(args.external_packages).toEqual([ - "package-with-dashes", - "package.with.dots", - "@scope/package-with-dashes", - ]); - }); - - it("should handle empty flag (no packages)", () => { - const parser = new ArgumentParser(); - parser.add_argument("--external-packages", { - nargs: "*", - help: "Additional packages to mark as external during bundling.", - }); - - const args = parser.parse_args(["--external-packages"]); - expect(args.external_packages).toEqual([]); - }); - - it("should handle flag not being used", () => { - const parser = new ArgumentParser(); - parser.add_argument("--external-packages", { - nargs: "*", - help: "Additional packages to mark as external during bundling.", - }); - - const args = parser.parse_args([]); - expect(args.external_packages).toBeUndefined(); - }); - - it("should handle mixed with other arguments", () => { - const parser = new ArgumentParser(); - parser.add_argument("--external-packages", { - nargs: "*", - help: "Additional packages to mark as external during bundling.", - }); - parser.add_argument("--verbose", { - action: "store_true", - help: "Verbose output", - }); - parser.add_argument("files", { - nargs: "*", - help: "Files to process", - }); - - const args = parser.parse_args([ - "--verbose", - "--external-packages", - "sqlite3", - "fsevents", - "--", - "file1.ts", - "file2.ts", - ]); - - expect(args.external_packages).toEqual(["sqlite3", "fsevents"]); - expect(args.verbose).toBe(true); - expect(args.files).toEqual(["file1.ts", "file2.ts"]); - }); - }); - - describe("Real-world scenarios", () => { - it("should handle database packages", () => { - const parser = new ArgumentParser(); - parser.add_argument("--external-packages", { - nargs: "*", - help: "Additional packages to mark as external during bundling.", - }); - - const args = parser.parse_args([ - "--external-packages", - "sqlite3", - "better-sqlite3", - "mysql2", - "pg", - ]); - - expect(args.external_packages).toEqual([ - "sqlite3", - "better-sqlite3", - "mysql2", - "pg", - ]); - }); - - it("should handle native modules", () => { - const parser = new ArgumentParser(); - parser.add_argument("--external-packages", { - nargs: "*", - help: "Additional packages to mark as external during bundling.", - }); - - const args = parser.parse_args([ - "--external-packages", - "sharp", - "canvas", - "node-sass", - "fsevents", - ]); - - expect(args.external_packages).toEqual([ - "sharp", - "canvas", - "node-sass", - "fsevents", - ]); - }); - - it("should handle problematic bundling packages", () => { - const parser = new ArgumentParser(); - parser.add_argument("--external-packages", { - nargs: "*", - help: "Additional packages to mark as external during bundling.", - }); - - const args = parser.parse_args([ - "--external-packages", - "@mapbox/node-pre-gyp", - "mock-aws-s3", - "aws-sdk", - "node-pre-gyp", - ]); - - expect(args.external_packages).toEqual([ - "@mapbox/node-pre-gyp", - "mock-aws-s3", - "aws-sdk", - "node-pre-gyp", - ]); - }); - }); -}); diff --git a/js/src/cli/util/pull.ts b/js/src/cli/util/pull.ts deleted file mode 100644 index 76d214237..000000000 --- a/js/src/cli/util/pull.ts +++ /dev/null @@ -1,394 +0,0 @@ -import { - Function as functionSchema, - type FunctionType as FunctionObject, - type SavedFunctionIdType as SavedFunctionId, - type ToolFunctionDefinitionType as ToolFunctionDefinition, -} from "../../generated_types"; -import { _internalGetGlobalState } from "../../logger"; -import { loadCLIEnv } from "./bundle"; -import { PullArgs } from "./types"; -import { warning } from "../../framework"; -import { z } from "zod/v3"; -import fs from "node:fs/promises"; -import util from "node:util"; -import { slugify } from "../../../util/string_util"; -import path from "node:path"; -import { runGitCommand } from "../../git-command"; -import { currentRepoPath } from "../../gitutil"; -import { isEmpty, loadPrettyXact, prettifyXact } from "../../../util/index"; -import { - ProjectNameIdMap, - toolFunctionDefinitionSchema, -} from "../../framework2"; -import pluralize from "pluralize"; - -export async function pullCommand(args: PullArgs) { - await loadCLIEnv(args); - - const loggerConn = _internalGetGlobalState().apiConn(); - const functions = await loggerConn.get_json("/v1/function", { - ...(args.project_id ? { project_id: args.project_id } : {}), - ...(args.project_name ? { project_name: args.project_name } : {}), - ...(args.slug ? { slug: args.slug } : {}), - ...(args.id ? { ids: [args.id] } : {}), - ...(args.version ? { version: loadPrettyXact(args.version) } : {}), - }); - const functionObjects = z - .object({ objects: z.array(z.unknown()) }) - .parse(functions); - - const projectNameToFunctions: Record = {}; - const projectNameIdMap = new ProjectNameIdMap(); - - for (const rawFunc of functionObjects.objects) { - const parsedFunc = functionSchema.safeParse(rawFunc); - if (!parsedFunc.success) { - const id = - typeof rawFunc === "object" && rawFunc && "id" in rawFunc - ? ` ${rawFunc.id}` - : ""; - // eslint-disable-next-line no-restricted-properties -- preserving intentional console usage. - console.warn( - warning(`Failed to parse function${id}: ${parsedFunc.error.message}`), - ); - continue; - } - - const func = parsedFunc.data; - const projectName = await projectNameIdMap.getName(func.project_id); - if (!projectNameToFunctions[projectName]) { - projectNameToFunctions[projectName] = []; - } - projectNameToFunctions[projectName].push(func); - } - - // eslint-disable-next-line no-restricted-properties -- preserving intentional console usage. - console.log("Found functions in the following projects:"); - for (const projectName of Object.keys(projectNameToFunctions)) { - // eslint-disable-next-line no-restricted-properties -- preserving intentional console usage. - console.log(` * ${projectName}`); - } - - const outputDir = args.output_dir ?? "./braintrust"; - await fs.mkdir(outputDir, { recursive: true }); - - const repoRoot = await currentRepoPath(); - const dirtyFileOutput = repoRoot - ? await runGitCommand(["diff", "--name-only", "-z", "HEAD"], { - cwd: repoRoot, - }) - : ""; - const dirtyFiles = new Set( - dirtyFileOutput - .split("\0") - .filter(Boolean) - .map((file) => path.resolve(repoRoot ?? ".", file)), - ); - - for (const projectName of Object.keys(projectNameToFunctions)) { - const projectFile = path.join( - outputDir, - `${slugify(projectName, { lower: true, strict: true, trim: true })}.ts`, - ); - const resolvedProjectFile = path.resolve(projectFile); - const fileExists = await fs.stat(projectFile).then( - () => true, - () => false, - ); - if (args.force) { - if (fileExists) { - // eslint-disable-next-line no-restricted-properties -- preserving intentional console usage. - console.warn( - warning( - `Overwriting ${doubleQuote(projectFile)} because --force is set.`, - ), - ); - } - } else if (dirtyFiles.has(resolvedProjectFile)) { - // eslint-disable-next-line no-restricted-properties -- preserving intentional console usage. - console.warn( - warning( - `Skipping project ${projectName} because ${doubleQuote(projectFile)} has uncommitted changes.`, - ), - ); - continue; - } else if (fileExists) { - if (!repoRoot) { - // eslint-disable-next-line no-restricted-properties -- preserving intentional console usage. - console.warn( - warning( - `Project ${projectName} already exists in ${doubleQuote(projectFile)}. Skipping since this is not a git repository...`, - ), - ); - continue; - } else { - // eslint-disable-next-line no-restricted-properties -- preserving intentional console usage. - console.warn( - warning( - `Project ${projectName} already exists in ${doubleQuote(projectFile)}. Overwriting...`, - ), - ); - } - } - - const projectFileContents = await makeProjectFile({ - projectName, - projectId: await projectNameIdMap.getId(projectName), - fileName: projectFile, - functions: projectNameToFunctions[projectName], - hasSpecifiedFunction: !!args.slug || !!args.id, - }); - await fs.writeFile(projectFile, projectFileContents || ""); - // eslint-disable-next-line no-restricted-properties -- preserving intentional console usage. - console.log(`Wrote ${projectName} to ${doubleQuote(projectFile)}`); - } -} - -async function makeProjectFile({ - projectName, - projectId, - fileName, - functions, - hasSpecifiedFunction, -}: { - projectName: string; - projectId: string; - fileName: string; - functions: FunctionObject[]; - hasSpecifiedFunction: boolean; -}) { - const varNames = {}; - const functionDefinitions = functions - .map((f) => - makeFunctionDefinition({ func: f, varNames, hasSpecifiedFunction }), - ) - .filter((f) => f !== null); - const fileDef = `// This file was automatically generated by braintrust pull. You can -// generate it again by running: -// $ braintrust pull --project-name ${doubleQuote(projectName)} -// Feel free to edit this file manually, but once you do, you should make sure to -// sync your changes with Braintrust by running: -// $ braintrust push ${doubleQuote(fileName)} - -import braintrust from "braintrust"; - -const project = braintrust.projects.create({ - id: ${doubleQuote(projectId)}, - name: ${doubleQuote(projectName)}, -}); - -${functionDefinitions.join("\n")} -`; - - const prettier = await getPrettierModule(); - if (prettier) { - try { - const formatted = prettier.format(fileDef, { - parser: "typescript", - }); - return formatted; - } catch (error) { - // eslint-disable-next-line no-restricted-properties -- preserving intentional console usage. - console.warn( - warning( - `Failed to format with prettier (${error instanceof Error ? error.message : error}). Using unformatted output.`, - ), - ); - } - } - return fileDef; -} - -function makeFunctionDefinition({ - func, - varNames, - hasSpecifiedFunction, -}: { - func: FunctionObject; - varNames: Record; - hasSpecifiedFunction: boolean; -}): string | null { - if (func.function_data.type !== "prompt") { - if (hasSpecifiedFunction) { - // eslint-disable-next-line no-restricted-properties -- preserving intentional console usage. - console.warn( - warning( - `Skipping function ${doubleQuote(func.name)} because it is not a prompt.`, - ), - ); - } - return null; - } - - const baseVarName = slugToVarName(func.slug); - let varName = baseVarName; - let suffix = 1; - while (varName in varNames) { - varName = `${varName}${suffix}`; - suffix++; - } - varNames[varName] = func.slug; - - if (!func.prompt_data || !func.prompt_data.prompt) { - // eslint-disable-next-line no-restricted-properties -- preserving intentional console usage. - console.warn( - warning( - `Prompt ${doubleQuote(func.name)} has an invalid (empty) prompt definition.`, - ), - ); - return null; - } - const objectType = "prompt"; - const prompt = func.prompt_data.prompt; - const promptContents = - prompt.type === "completion" - ? `prompt: ${doubleQuote(prompt.content)}` - : `messages: ${safeStringify(prompt.messages).trimStart()}`; - - const rawToolsParsed = - prompt.type === "chat" && prompt.tools && prompt.tools.length > 0 - ? z - .array(toolFunctionDefinitionSchema) - .safeParse(JSON.parse(prompt.tools)) - : undefined; - - if (rawToolsParsed && !rawToolsParsed.success) { - // eslint-disable-next-line no-restricted-properties -- preserving intentional console usage. - console.warn( - warning( - `Prompt ${doubleQuote(func.name)} has an invalid tools definition: ${rawToolsParsed.error.message}. Skipping...`, - ), - ); - return null; - } - - const rawTools = rawToolsParsed ? rawToolsParsed.data : []; - - const { model, params } = func.prompt_data.options ?? {}; - - const paramsString = - params && Object.keys(params).length > 0 - ? `params: ${safeStringify(params).trimStart()},` - : ""; - - const tools: (SavedFunctionId | ToolFunctionDefinition)[] = [ - ...(func.prompt_data.tool_functions ?? []), - ...rawTools, - ]; - - const toolsString = - tools.length > 0 ? `tools: ${safeStringify(tools).trimStart()},` : ""; - - return `export const ${varName} = project.${pluralize(objectType)}.create({ - id: ${doubleQuote(func.id)}, - name: ${doubleQuote(func.name)}, - slug: ${doubleQuote(func.slug)}, - version: ${doubleQuote(prettifyXact(func._xact_id))}, ${printOptionalField("description", func.description)}${printOptionalField("model", model)} -${indent(promptContents, 2)}, -${indent(paramsString, 2)} -${indent(toolsString, 2)} -}); -`; -} - -function doubleQuote(str: string) { - return JSON.stringify(str); -} - -function slugToVarName(slug: string) { - let varName = slug.replace(/^[^a-zA-Z_$]|[^a-zA-Z0-9_$]/g, "_"); - varName = varName.replace(/_([a-z])/g, (_, letter) => letter.toUpperCase()); - varName = varName.charAt(0).toLowerCase() + varName.slice(1); - return varName; -} - -function indent(str: string, numSpaces: number) { - return str.replace(/^/gm, " ".repeat(numSpaces)); -} - -function printOptionalField( - fieldName: string, - fieldValue: string | undefined | null, -) { - return !isEmpty(fieldValue) - ? ` - ${fieldName}: ${doubleQuote(fieldValue)},` - : ""; -} - -let prettierImportAttempted = false; - -let prettierModule: typeof import("prettier") | undefined = undefined; - -async function getPrettierModule() { - if (!prettierModule && !prettierImportAttempted) { - prettierImportAttempted = true; - - try { - // First try require() which is more stable in npx environments - prettierModule = require("prettier"); - } catch { - try { - // Fallback to dynamic import with error boundary - const importWithTimeout = () => { - return new Promise((resolve, reject) => { - let resolved = false; - - // Set a timeout to prevent infinite hanging - const timeoutId = setTimeout(() => { - if (!resolved) { - resolved = true; - reject(new Error("Prettier import timeout")); - } - }, 3000); - - import("prettier") - .then((module) => { - if (!resolved) { - resolved = true; - clearTimeout(timeoutId); - resolve(module); - } - }) - .catch((error) => { - if (!resolved) { - resolved = true; - clearTimeout(timeoutId); - reject(error); - } - }); - }); - }; - - prettierModule = await importWithTimeout(); - } catch { - // eslint-disable-next-line no-restricted-properties -- preserving intentional console usage. - console.warn( - warning( - "Failed to load prettier module. Will not use prettier to format output.", - ), - ); - prettierModule = undefined; - } - } - } - return prettierModule; -} - -function safeStringify(obj: unknown): string { - try { - return JSON.stringify(obj, null, 2); - } catch (error) { - // Fallback for circular references or other JSON.stringify issues - try { - return util.inspect(obj, { - depth: 5, - maxStringLength: 1000, - breakLength: 80, - compact: false, - }); - } catch { - return `[Object: Unable to serialize - ${error instanceof Error ? error.message : error}]`; - } - } -} diff --git a/js/src/cli/util/types.ts b/js/src/cli/util/types.ts deleted file mode 100644 index 2909a607e..000000000 --- a/js/src/cli/util/types.ts +++ /dev/null @@ -1,51 +0,0 @@ -import { type IfExistsType as IfExists } from "../../generated_types"; - -export interface CommonArgs { - /** @deprecated Use debug_logging: "debug" for detailed troubleshooting output. */ - verbose: boolean; - debug_logging?: "error" | "warn" | "info" | "debug"; -} - -export interface AuthArgs { - api_key?: string; - org_name?: string; - app_url?: string; - env_file?: string; -} - -export interface CompileArgs { - tsconfig?: string; - terminate_on_failure: boolean; - external_packages?: string[]; -} - -export interface RunArgs extends CommonArgs, AuthArgs, CompileArgs { - files: string[]; - watch: boolean; - list: boolean; - jsonl: boolean; - filter?: string[]; - no_send_logs: boolean; - no_progress_bars: boolean; - bundle: boolean; - push: boolean; - dev: boolean; - dev_host: string; - dev_port: number; - dev_org_name?: string; -} - -export interface BundleArgs extends CommonArgs, AuthArgs, CompileArgs { - files: string[]; - if_exists: IfExists; -} - -export interface PullArgs extends CommonArgs, AuthArgs { - output_dir: string; - project_name?: string; - project_id?: string; - id?: string; - slug?: string; - version?: string; - force: boolean; -} diff --git a/js/src/durable-eval.ts b/js/src/durable-eval.ts index 7a0c45e70..c27f63d26 100644 --- a/js/src/durable-eval.ts +++ b/js/src/durable-eval.ts @@ -852,7 +852,6 @@ async function startDurableEval< stop: () => undefined, increment: () => undefined, }, - [], undefined, parameters, true, diff --git a/js/src/exports.ts b/js/src/exports.ts index e8bac9cc2..d39c15459 100644 --- a/js/src/exports.ts +++ b/js/src/exports.ts @@ -253,7 +253,6 @@ export type { EvalScorer, EvalClassifier, EvaluatorDef, - EvaluatorFile, ReporterBody, SpanContext, } from "./framework"; @@ -263,9 +262,6 @@ export { Eval, EvalResultWithSummary, Reporter, - buildLocalSummary, - reportFailures, - runEvaluator, defaultErrorScoreHandler, } from "./framework"; @@ -286,7 +282,6 @@ export { DatasetPipeline } from "./dataset-pipeline"; export type { CodeOpts, CreateProjectOpts, - FunctionEvent, PromptOpts, ScorerOpts, } from "./framework2"; @@ -295,12 +290,10 @@ export { CodeFunction, CodePrompt, Project, - ProjectNameIdMap, PromptBuilder, ScorerBuilder, ToolBuilder, projects, - toolFunctionDefinitionSchema, } from "./framework2"; export { @@ -334,21 +327,8 @@ export { addAzureBlobHeaders, LazyValue } from "./util"; export { AttachmentReference } from "./generated_types"; -export type { - EvaluatorManifest, - EvalParameterSerializedSchema, - EvaluatorDefinition, - EvaluatorDefinitions, - ParametersSource, -} from "../dev/types"; - export type { EvalParameters } from "./eval-parameters"; -export { - evaluatorDefinitionSchema, - evaluatorDefinitionsSchema, -} from "../dev/types"; - // Auto-instrumentation configuration export { configureInstrumentation } from "./instrumentation"; export { braintrustFlueInstrumentation } from "./instrumentation"; diff --git a/js/src/framework.test.ts b/js/src/framework.test.ts index 4c86a8235..cb905aad7 100644 --- a/js/src/framework.test.ts +++ b/js/src/framework.test.ts @@ -54,7 +54,6 @@ test("meta (write) is passed to task", async () => { scores: [], }, new NoopProgressReporter(), - [], undefined, undefined, true, @@ -92,7 +91,6 @@ test("metadata (read/write) is passed to task", async () => { scores: [], }, new NoopProgressReporter(), - [], undefined, undefined, true, @@ -132,7 +130,6 @@ test("expected (read/write) is passed to task", async () => { scores: [], }, new NoopProgressReporter(), - [], undefined, undefined, true, @@ -175,7 +172,6 @@ test("EvalCase id and tags are passed to scorers", async () => { ], }, new NoopProgressReporter(), - [], undefined, undefined, true, @@ -259,7 +255,6 @@ describe("runEvaluator", () => { scores: [], }, new NoopProgressReporter(), - [], undefined, ); @@ -292,7 +287,6 @@ describe("runEvaluator", () => { scores: [], }, new NoopProgressReporter(), - [], undefined, ); @@ -331,7 +325,6 @@ describe("runEvaluator", () => { scores: [], }, new NoopProgressReporter(), - [], undefined, ); @@ -369,7 +362,6 @@ describe("runEvaluator", () => { scores: [], }, new NoopProgressReporter(), - [], (event) => streamEvents.push(event), ); @@ -408,7 +400,6 @@ describe("runEvaluator", () => { scores: [], }, new NoopProgressReporter(), - [], undefined, ); @@ -432,7 +423,6 @@ describe("runEvaluator", () => { ), }, new NoopProgressReporter(), - [], undefined, ); @@ -459,7 +449,6 @@ describe("runEvaluator", () => { errorScoreHandler: defaultErrorScoreHandler, }, new NoopProgressReporter(), - [], undefined, ); @@ -488,7 +477,6 @@ describe("runEvaluator", () => { errorScoreHandler: defaultErrorScoreHandler, }, new NoopProgressReporter(), - [], undefined, ); @@ -521,7 +509,6 @@ describe("runEvaluator", () => { errorScoreHandler: () => undefined, }, new NoopProgressReporter(), - [], undefined, ); @@ -546,7 +533,6 @@ describe("runEvaluator", () => { errorScoreHandler: () => ({ error_score: 1 }), }, new NoopProgressReporter(), - [], undefined, ); @@ -579,7 +565,6 @@ describe("runEvaluator", () => { }, }, new NoopProgressReporter(), - [], undefined, undefined, true, @@ -624,7 +609,6 @@ describe("runEvaluator", () => { maxConcurrency: 1, }, new NoopProgressReporter(), - [], undefined, ), ).rejects.toThrow(new InternalAbortError("Evaluator timed out")); @@ -673,7 +657,6 @@ describe("runEvaluator", () => { maxConcurrency: 1, }, new NoopProgressReporter(), - [], undefined, ), ).rejects.toThrow(new InternalAbortError("Evaluator aborted")); @@ -708,7 +691,6 @@ describe("runEvaluator", () => { scores: [], }, new NoopProgressReporter(), - [], undefined, ); @@ -740,7 +722,6 @@ describe("runEvaluator", () => { signal: abortController.signal, }, new NoopProgressReporter(), - [], undefined, ); @@ -769,7 +750,6 @@ test("trialIndex is passed to task", async () => { trialCount: 3, }, new NoopProgressReporter(), - [], undefined, undefined, true, @@ -811,7 +791,6 @@ test("trialIndex with multiple inputs", async () => { trialCount: 2, }, new NoopProgressReporter(), - [], undefined, undefined, true, @@ -857,7 +836,6 @@ test("per-input trialCount overrides global trialCount", async () => { trialCount: 2, }, new NoopProgressReporter(), - [], undefined, undefined, true, @@ -904,7 +882,6 @@ test("per-input trialCount works without global trialCount", async () => { scores: [], }, new NoopProgressReporter(), - [], undefined, undefined, true, @@ -1071,7 +1048,6 @@ test("runEvaluator forwards baseExperimentId to summary", async () => { baseExperimentId: "base-exp-id", }, new NoopProgressReporter(), - [], undefined, undefined, true, @@ -1114,7 +1090,6 @@ test("runEvaluator forwards persisted baseExperimentName id to summary", async ( baseExperimentName: "base-exp", }, new NoopProgressReporter(), - [], undefined, undefined, true, @@ -1156,7 +1131,6 @@ test("tags can be appended and logged to root span", async () => { summarizeScores: false, }, new NoopProgressReporter(), - [], undefined, undefined, true, @@ -1205,7 +1179,6 @@ test.each([ summarizeScores: false, }, new NoopProgressReporter(), - [], undefined, undefined, true, @@ -1245,7 +1218,6 @@ test("tags are persisted with a failing scorer", async () => { summarizeScores: false, }, new NoopProgressReporter(), - [], undefined, undefined, true, @@ -1278,7 +1250,6 @@ test("tags remain empty when not set", async () => { summarizeScores: false, }, new NoopProgressReporter(), - [], undefined, undefined, true, @@ -1313,7 +1284,6 @@ test("scorer spans have purpose='scorer' attribute", async () => { ], }, new NoopProgressReporter(), - [], undefined, undefined, true, diff --git a/js/src/framework.ts b/js/src/framework.ts index 42d4e09ff..9a463533b 100644 --- a/js/src/framework.ts +++ b/js/src/framework.ts @@ -121,7 +121,7 @@ export type EvalTask< hooks: EvalHooks, ) => Output); -export type TaskProgressEvent = Omit< +type TaskProgressEvent = Omit< SSEProgressEventData, "id" | "origin" | "object_type" | "name" >; @@ -404,7 +404,7 @@ export class EvalResultWithSummary< } } -export type { ReporterBody, ReporterDef } from "./reporters/types"; +export type { ReporterBody } from "./reporters/types"; async function getPersistedBaseExperimentId( experiment: Experiment, @@ -435,7 +435,7 @@ export type EvaluatorDef< evalName: string; } & Evaluator; -export type EvaluatorFile = { +type EvaluatorFile = { functions: CodeFunction< unknown, unknown, @@ -800,7 +800,7 @@ export async function Eval< if (typeof options.reporter === "string") { throw new Error( - "Must specify a reporter object, not a name. Can only specify reporter names when running 'braintrust eval'", + "Must specify a reporter object, not a name. Can only specify reporter names when running 'bt eval'", ); } @@ -847,7 +847,6 @@ export async function Eval< null, evalDef, progressReporter, - [], options.stream, options.parameters, shouldCollectResults, @@ -860,7 +859,6 @@ export async function Eval< experiment, evalDef, progressReporter, - [], options.stream, options.parameters, shouldCollectResults, @@ -903,64 +901,7 @@ export function Reporter( return ret; } -export interface Filter { - path: string[]; - pattern: RegExp; -} - -function serializeJSONWithPlainString(v: unknown) { - if (typeof v === "string") { - return v; - } else { - return JSON.stringify(v); - } -} - -function deserializePlainStringAsJSON(s: string) { - try { - return { value: JSON.parse(s), error: undefined }; - } catch (e) { - return { value: s, error: e }; - } -} - -export function parseFilters(filters: string[]): Filter[] { - const result: Filter[] = []; - for (const f of filters) { - const equalsIdx = f.indexOf("="); - if (equalsIdx === -1) { - throw new Error(`Invalid filter ${f}`); - } - const [path, value] = [f.slice(0, equalsIdx), f.slice(equalsIdx + 1)]; - let deserializedValue = deserializePlainStringAsJSON(value).value; - if (typeof deserializedValue !== "string") { - deserializedValue = value; // Just fall back to the original input - } - result.push({ - path: path.split("."), - pattern: new RegExp(deserializedValue), - }); - } - return result; -} - -function evaluateFilter(object: unknown, filter: Filter) { - const { path, pattern } = filter; - const key = path.reduce( - (acc, p) => - typeof acc === "object" && acc !== null - ? // eslint-disable-next-line @typescript-eslint/consistent-type-assertions - (acc as Record)[p] - : undefined, - object, - ); - if (key === undefined) { - return false; - } - return pattern.test(serializeJSONWithPlainString(key)); -} - -export function scorerName( +function scorerName( // eslint-disable-next-line @typescript-eslint/no-explicit-any scorer: EvalScorer, scorer_idx: number, @@ -1207,7 +1148,6 @@ export async function runEvaluator( // eslint-disable-next-line @typescript-eslint/no-explicit-any evaluator: EvaluatorDef, progressReporter: ProgressReporter, - filters: Filter[], stream: ((data: SSEProgressEventData) => void) | undefined, parameters?: InferParameters, collectResults = true, @@ -1223,7 +1163,6 @@ export async function runEvaluator( experiment, evaluator, progressReporter, - filters, stream, parameters, collectResults, @@ -1246,7 +1185,6 @@ async function runEvaluatorInternal( // eslint-disable-next-line @typescript-eslint/no-explicit-any evaluator: EvaluatorDef, progressReporter: ProgressReporter, - filters: Filter[], stream: ((data: SSEProgressEventData) => void) | undefined, parameters: InferParameters | undefined, collectResults: boolean, @@ -1616,9 +1554,6 @@ async function runEvaluatorInternal( if (cancelled) { break; } - if (!filters.every((f) => evaluateFilter(datum, f))) { - continue; - } const trialCount = datum.trialCount ?? evaluator.trialCount ?? 1; for (let trialIndex = 0; trialIndex < trialCount; trialIndex++) { if (cancelled) { @@ -1756,10 +1691,9 @@ async function runEvaluatorInternal( } } -export const error = (text: string) => `Error: ${text}`; -export const warning = (text: string) => `Warning: ${text}`; +const warning = (text: string) => `Warning: ${text}`; -export function logError(e: unknown, verbose: boolean) { +function logError(e: unknown, verbose: boolean) { if (!verbose) { // eslint-disable-next-line no-restricted-properties -- preserving intentional console usage. console.error(`${e}`); @@ -1829,12 +1763,7 @@ export function buildLocalSummary( }; } -export function reportFailures< - Input, - Output, - Expected, - Metadata extends BaseMetadata, ->( +function reportFailures( evaluator: EvaluatorDef, failingResults: EvalResult[], { verbose, jsonl }: ReporterOpts, diff --git a/js/src/framework2.ts b/js/src/framework2.ts index 47994f5f1..f9220b6e1 100644 --- a/js/src/framework2.ts +++ b/js/src/framework2.ts @@ -9,7 +9,6 @@ import { type SavedFunctionIdType as SavedFunctionId, type PromptBlockDataType as PromptBlockData, type PromptDataType as PromptData, - ToolFunctionDefinition as toolFunctionDefinitionSchema, type ToolFunctionDefinitionType as ToolFunctionDefinition, FunctionData as functionDataSchema, Project as projectSchema, @@ -21,7 +20,6 @@ import { login, Prompt, PromptRowWithId, - RemoteEvalParameters, } from "./logger"; import type { BaseFnOpts, GenericFunction } from "./framework-types"; import type { EvalParameters } from "./eval-parameters"; @@ -30,14 +28,12 @@ import { type PromptDefinition, } from "./prompt-schemas"; import { zodToJsonSchema } from "./zod/utils"; -import type { - ParametersSchema, - StaticParametersSchema, - SerializedParametersContainer, -} from "../dev/types"; - -export { toolFunctionDefinitionSchema }; -// ToolFunctionDefinition exported as type-only from main index to avoid namespace issues +type ParametersSchema = { + type: "object"; + properties: Record>; + required?: string[]; + additionalProperties?: boolean; +}; // Safe access to __filename (only exists in Node.js CJS) const currentFilename = @@ -120,7 +116,7 @@ export class Project { async publish() { if (globalThis._lazy_load) { // eslint-disable-next-line no-restricted-properties -- preserving intentional console usage. - console.warn("publish() is a no-op when running `braintrust push`."); + console.warn("publish() is a no-op when running `bt push`."); return; } await login(); @@ -129,7 +125,7 @@ export class Project { if (this._publishableCodeFunctions.length > 0) { // eslint-disable-next-line no-restricted-properties -- preserving intentional console usage. console.warn( - "Code functions cannot be published directly. Use `braintrust push` instead.", + "Code functions cannot be published directly. Use `bt push` instead.", ); } if (this._publishablePrompts.length > 0) { @@ -710,53 +706,6 @@ class ParametersBuilder { } } -export function serializeEvalParametersToStaticParametersSchema( - parameters: EvalParameters, -): StaticParametersSchema { - return Object.fromEntries( - Object.entries(parameters).map(([name, value]) => { - if ("type" in value && value.type === "prompt") { - return [ - name, - { - type: "prompt", - default: value.default - ? promptDefinitionToPromptData(value.default) - : undefined, - description: value.description, - }, - ]; - } else if ("type" in value && value.type === "model") { - return [ - name, - { - type: "model", - default: value.default, - description: value.description, - }, - ]; - } else { - // Since this schema is bundled, it won't pass an instanceof check. For - // some reason, aliasing it to `z.ZodSchema` leads to `error TS2589: - // Type instantiation is excessively deep and possibly infinite.` So - // just using `any` to turn off the typesystem. - // - // eslint-disable-next-line @typescript-eslint/consistent-type-assertions - const schemaObj = zodToJsonSchema(value as unknown as z.ZodType); - return [ - name, - { - type: "data", - schema: schemaObj, - default: schemaObj.default, - description: schemaObj.description, - }, - ]; - } - }), - ); -} - function serializeEvalParameterstoParametersSchema( parameters: EvalParameters, ): ParametersSchema { @@ -827,32 +776,7 @@ function getDefaultDataFromParametersSchema( ); } -export function serializeRemoteEvalParametersContainer( - parameters: EvalParameters | RemoteEvalParameters, -): SerializedParametersContainer { - if (RemoteEvalParameters.isParameters(parameters)) { - return { - type: "braintrust.parameters", - // eslint-disable-next-line @typescript-eslint/consistent-type-assertions - schema: parameters.schema as ParametersSchema, - source: { - parametersId: parameters.id, - slug: parameters.slug, - name: parameters.name, - projectId: parameters.projectId, - version: parameters.version, - }, - }; - } - - return { - type: "braintrust.staticParameters", - schema: serializeEvalParametersToStaticParametersSchema(parameters), - source: null, - }; -} - -export interface FunctionEvent { +interface FunctionEvent { project_id: string; slug: string; name: string; @@ -866,7 +790,7 @@ export interface FunctionEvent { environments?: { slug: string }[]; } -export class ProjectNameIdMap { +class ProjectNameIdMap { private nameToId: Record = {}; private idToName: Record = {}; diff --git a/js/src/gitutil.ts b/js/src/gitutil.ts index 7b23999af..8e318ac98 100644 --- a/js/src/gitutil.ts +++ b/js/src/gitutil.ts @@ -171,7 +171,7 @@ export async function getRepoInfo(settings?: GitMetadataSettings) { return sanitized; } -export async function currentRepoPath(): Promise { +async function currentRepoPath(): Promise { return await attempt(async () => (await runGitCommand(["rev-parse", "--show-toplevel"])).trim(), ); diff --git a/js/src/imports.test.ts b/js/src/imports.test.ts index 96a4851d5..c2d53c767 100644 --- a/js/src/imports.test.ts +++ b/js/src/imports.test.ts @@ -2,88 +2,7 @@ import { describe, it, expect } from "vitest"; import * as fs from "fs"; import * as path from "path"; -describe("CLI import restrictions", () => { - it("should not import from cli directory in non-cli code", () => { - const srcDir = path.join(__dirname); - const violations: string[] = []; - - function walkDirectory(dir: string) { - const entries = fs.readdirSync(dir, { withFileTypes: true }); - - for (const entry of entries) { - const fullPath = path.join(dir, entry.name); - const relativePath = path.relative(srcDir, fullPath); - - // Skip CLI directory and test files - if (relativePath.startsWith("cli/") || relativePath === "cli") { - continue; - } - - if (entry.isDirectory() && entry.name === "node_modules") { - continue; - } - - if (entry.isDirectory()) { - walkDirectory(fullPath); - } else if ( - entry.isFile() && - (entry.name.endsWith(".ts") || entry.name.endsWith(".tsx")) && - !entry.name.endsWith(".d.ts") && - !entry.name.endsWith(".d.tsx") - ) { - checkFileForCliImports(fullPath, relativePath); - } - } - } - - function checkFileForCliImports(filePath: string, relativePath: string) { - const content = fs.readFileSync(filePath, "utf-8"); - const lines = content.split("\n"); - - lines.forEach((line, index) => { - // Check for import statements that reference CLI - const importMatch = - line.match(/from\s+["']([^"']+)["']/) || - line.match(/import\s*\(\s*["']([^"']+)["']\s*\)/); - - if (importMatch) { - const importPath = importMatch[1]; - - // Check if the import path references the CLI directory - // Only match actual /cli/ directory paths, not package names containing "cli" - // (e.g., @temporalio/client should not match) - if ( - importPath.includes("/cli/") || - importPath === "./cli" || - importPath === "../cli" || - importPath.endsWith("/cli") || - importPath.match(/^\.\.\/.*\/cli$/) || - importPath.match(/^\.\.\/.*\/cli\//) - ) { - violations.push( - `${relativePath}:${index + 1} - Illegal import from CLI: "${importPath}"`, - ); - } - } - }); - } - - walkDirectory(srcDir); - - if (violations.length > 0) { - const message = [ - "Found illegal imports from CLI directory in SDK code:", - "", - ...violations, - "", - "SDK code (src/**) must not import from CLI code (src/cli/**).", - "CLI code can import from SDK code, but not vice versa.", - ].join("\n"); - - expect.fail(message); - } - }); - +describe("Import restrictions", () => { it("should not allow eslint-disable comments for no-restricted-imports", () => { const srcDir = path.join(__dirname); const violations: string[] = []; @@ -95,11 +14,6 @@ describe("CLI import restrictions", () => { const fullPath = path.join(dir, entry.name); const relativePath = path.relative(srcDir, fullPath); - // Skip CLI directory and test files - if (relativePath.startsWith("cli/") || relativePath === "cli") { - continue; - } - if (entry.isDirectory() && entry.name === "node_modules") { continue; } @@ -153,7 +67,7 @@ describe("CLI import restrictions", () => { ...violations, "", "Disabling the no-restricted-imports rule is not allowed.", - "This rule prevents SDK code from importing CLI code and cannot be bypassed.", + "This rule protects SDK module boundaries and cannot be bypassed.", "If you believe you have a legitimate need for this import, please discuss with the team.", ].join("\n"); @@ -172,11 +86,6 @@ describe("CLI import restrictions", () => { const fullPath = path.join(dir, entry.name); const relativePath = path.relative(srcDir, fullPath); - // Skip the cli directory - CLI code is allowed to use require() and dynamic imports - if (entry.isDirectory() && entry.name === "cli") { - continue; - } - // Skip node_modules directories (test fixture deps, not SDK source) if (entry.isDirectory() && entry.name === "node_modules") { continue; diff --git a/js/src/node/index.ts b/js/src/node/index.ts index ba75fa586..60e1efe09 100644 --- a/js/src/node/index.ts +++ b/js/src/node/index.ts @@ -42,10 +42,10 @@ * }); * ``` * - * Finally, run the script with `npx braintrust eval hello.eval.ts`. + * Finally, run the script with `npx bt eval hello.eval.ts`. * * ```bash - * BRAINTRUST_API_KEY= npx braintrust eval hello.eval.ts + * BRAINTRUST_API_KEY= npx bt eval hello.eval.ts * ``` * * @module braintrust diff --git a/js/src/parameters.test.ts b/js/src/parameters.test.ts index b22953561..69576dd9a 100644 --- a/js/src/parameters.test.ts +++ b/js/src/parameters.test.ts @@ -35,7 +35,6 @@ test("parameters are passed to task", async () => { }, }, new NoopProgressReporter(), - [], undefined, { prefix: "start:", suffix: ":end" }, true, @@ -81,7 +80,6 @@ test("prompt parameter is passed correctly", async () => { }, }, new NoopProgressReporter(), - [], undefined, undefined, true, @@ -163,7 +161,6 @@ test("custom parameter values override defaults", async () => { }, }, new NoopProgressReporter(), - [], undefined, { prefix: "custom:", @@ -195,7 +192,6 @@ test("array parameter is handled correctly", async () => { }, }, new NoopProgressReporter(), - [], undefined, undefined, true, @@ -234,7 +230,6 @@ test("object parameter is handled correctly", async () => { }, }, new NoopProgressReporter(), - [], undefined, undefined, true, @@ -265,7 +260,6 @@ test("model parameter defaults to configured value", async () => { }, }, new NoopProgressReporter(), - [], undefined, undefined, true, @@ -293,7 +287,6 @@ test("model parameter is required when default is missing", async () => { }, }, new NoopProgressReporter(), - [], undefined, undefined, true, diff --git a/js/src/reporters/types.ts b/js/src/reporters/types.ts index f68724bf6..a85be9bdb 100644 --- a/js/src/reporters/types.ts +++ b/js/src/reporters/types.ts @@ -32,7 +32,7 @@ export interface ReporterBody { /** * A function that takes all evaluator results and returns a boolean indicating - * whether the run was successful. If you return false, the `braintrust eval` + * whether the run was successful. If you return false, the `bt eval` * command will exit with a non-zero status code. * * @param reports diff --git a/js/src/zod/zod-serialization-test-shared.ts b/js/src/zod/zod-serialization-test-shared.ts deleted file mode 100644 index a78f4b708..000000000 --- a/js/src/zod/zod-serialization-test-shared.ts +++ /dev/null @@ -1,70 +0,0 @@ -/** - * Shared test expectations for zod serialization tests - * - * This module contains expected JSON Schema outputs used by both v3 and v4 - * serialization tests to ensure consistent behavior across versions. - */ - -/** - * Expected JSON schema for string parameter with description and default - */ - -export const EXPECTED_STRING_SCHEMA = { - type: "string", - description: "The instructions for the agent", - default: "You are a helpful assistant.", - $schema: "http://json-schema.org/draft-07/schema#", -}; - -/** - * Expected JSON schema for number parameter with constraints - */ - -export const EXPECTED_NUMBER_SCHEMA = { - type: "number", - minimum: 0, - maximum: 2, - description: "Temperature for LLM", - default: 0.7, - $schema: "http://json-schema.org/draft-07/schema#", -}; - -/** - * Expected JSON schema for object parameter - */ - -export const EXPECTED_OBJECT_SCHEMA = { - type: "object", - properties: { - model: { type: "string" }, - maxTokens: { type: "number" }, - }, - additionalProperties: false, - required: ["model"], - description: "Configuration object", - $schema: "http://json-schema.org/draft-07/schema#", -}; - -/** - * Expected JSON schema for enum parameter - */ - -export const EXPECTED_ENUM_SCHEMA = { - type: "string", - enum: ["fast", "accurate", "balanced"], - description: "Processing mode", - default: "balanced", - $schema: "http://json-schema.org/draft-07/schema#", -}; - -/** - * Expected JSON schema for array parameter - */ - -export const EXPECTED_ARRAY_SCHEMA = { - type: "array", - items: { type: "string" }, - description: "Tags for filtering", - default: ["default"], - $schema: "http://json-schema.org/draft-07/schema#", -}; diff --git a/js/src/zod/zod-v3-serialization.test.ts b/js/src/zod/zod-v3-serialization.test.ts deleted file mode 100644 index c4d1097d2..000000000 --- a/js/src/zod/zod-v3-serialization.test.ts +++ /dev/null @@ -1,234 +0,0 @@ -/** - * Tests for zod v3 compatibility with serializeEvalParametersToStaticParametersSchema - * - * This file tests serializeEvalParametersToStaticParametersSchema specifically with zod v3. - * See zod-v4-serialization.test.ts for zod v4 specific tests. - */ - -import { test, describe, beforeEach, expect } from "vitest"; -import * as zodModule from "zod"; -import { z } from "zod"; -import { - EXPECTED_STRING_SCHEMA, - EXPECTED_NUMBER_SCHEMA, - EXPECTED_OBJECT_SCHEMA, - EXPECTED_ENUM_SCHEMA, - EXPECTED_ARRAY_SCHEMA, -} from "./zod-serialization-test-shared"; -import { - serializeEvalParametersToStaticParametersSchema, - serializeRemoteEvalParametersContainer, -} from "../framework2"; -import { - serializedParametersContainerSchema, - evaluatorDefinitionSchema, -} from "../../dev/types"; -// Detect which zod version is installed by checking for v4-specific properties -function getInstalledZodVersion(): 3 | 4 { - const testSchema = zodModule.z.string(); - // eslint-disable-next-line @typescript-eslint/no-explicit-any - return "_zod" in (testSchema as any) ? 4 : 3; -} - -describe("serializeEvalParametersToStaticParametersSchema with Zod v3", () => { - beforeEach(() => { - const version = getInstalledZodVersion(); - expect(version).toBe(3); - }); - - test("string schema serializes correctly", () => { - const parameters = { - instructions: z - .string() - .describe("The instructions for the agent") - .default("You are a helpful assistant."), - }; - - const result = serializeEvalParametersToStaticParametersSchema(parameters); - console.log(result); - expect(result.instructions).toBeDefined(); - expect(result.instructions.type).toBe("data"); - expect( - result.instructions.type === "data" && result.instructions.schema, - ).toStrictEqual(EXPECTED_STRING_SCHEMA); - expect(result.instructions.description).toBe( - "The instructions for the agent", - ); - expect(result.instructions.default).toBe("You are a helpful assistant."); - }); - - test("number schema serializes correctly", () => { - const parameters = { - temperature: z - .number() - .min(0) - .max(2) - .describe("Temperature for LLM") - .default(0.7), - }; - - const result = serializeEvalParametersToStaticParametersSchema(parameters); - - result.temperature.type; - - expect(result.temperature.type).toBe("data"); - expect( - result.temperature.type === "data" && result.temperature.schema, - ).toStrictEqual(EXPECTED_NUMBER_SCHEMA); - expect(result.temperature.description).toBe("Temperature for LLM"); - expect(result.temperature.default).toBe(0.7); - }); - - test("object schema serializes correctly", () => { - const parameters = { - config: z - .object({ - model: z.string(), - maxTokens: z.number().optional(), - }) - .describe("Configuration object"), - }; - - const result = serializeEvalParametersToStaticParametersSchema(parameters); - - expect(result.config.type).toBe("data"); - expect(result.config.type === "data" && result.config.schema).toStrictEqual( - EXPECTED_OBJECT_SCHEMA, - ); - expect(result.config.description).toBe("Configuration object"); - }); - - test("enum schema serializes correctly", () => { - const parameters = { - mode: z - .enum(["fast", "accurate", "balanced"]) - .describe("Processing mode") - .default("balanced"), - }; - - const result = serializeEvalParametersToStaticParametersSchema(parameters); - - expect(result.mode.type).toBe("data"); - expect(result.mode.type === "data" && result.mode.schema).toStrictEqual( - EXPECTED_ENUM_SCHEMA, - ); - expect(result.mode.description).toBe("Processing mode"); - expect(result.mode.default).toBe("balanced"); - }); - - test("array schema serializes correctly", () => { - const parameters = { - tags: z - .array(z.string()) - .describe("Tags for filtering") - .default(["default"]), - }; - - const result = serializeEvalParametersToStaticParametersSchema(parameters); - - expect(result.tags.type).toBe("data"); - expect(result.tags.type === "data" && result.tags.schema).toStrictEqual( - EXPECTED_ARRAY_SCHEMA, - ); - expect(result.tags.description).toBe("Tags for filtering"); - expect(result.tags.default).toEqual(["default"]); - }); - - test("model parameter serializes correctly", () => { - const parameters = { - model: { - type: "model" as const, - description: "Model to use", - default: "gpt-5-mini", - }, - }; - - const result = serializeEvalParametersToStaticParametersSchema(parameters); - - expect(result.model.type).toBe("model"); - expect(result.model.description).toBe("Model to use"); - expect(result.model.default).toBe("gpt-5-mini"); - }); -}); - -describe("serializeRemoteEvalParametersContainer with Zod v3", () => { - beforeEach(() => { - const version = getInstalledZodVersion(); - expect(version).toBe(3); - }); - - test("local parameters are serialized with legacyParameters type", () => { - const parameters = { - instructions: z - .string() - .describe("The instructions for the agent") - .default("You are a helpful assistant."), - }; - - const result = serializeRemoteEvalParametersContainer(parameters); - - expect(result.type).toBe("braintrust.staticParameters"); - expect(result.source).toBeNull(); - if (result.type === "braintrust.staticParameters") { - expect(result.schema.instructions).toBeDefined(); - expect(result.schema.instructions.type).toBe("data"); - } - }); -}); - -describe("serializedParametersContainerSchema parsing", () => { - const schema = { - model: { - type: "model" as const, - description: "Model to use", - default: "gpt-5-mini", - }, - }; - - test("parses braintrust.staticParameters with source: null", () => { - const result = serializedParametersContainerSchema.safeParse({ - type: "braintrust.staticParameters", - schema, - source: null, - }); - expect(result.success).toBe(true); - }); - - test("parses braintrust.staticParameters with source omitted", () => { - const result = serializedParametersContainerSchema.safeParse({ - type: "braintrust.staticParameters", - schema, - }); - expect(result.success).toBe(true); - }); -}); - -describe("evaluatorDefinitionSchema parameters nullish", () => { - test("allows parameters: null", () => { - const result = evaluatorDefinitionSchema.safeParse({ - parameters: null, - }); - expect(result.success).toBe(true); - if (result.success) { - expect(result.data.parameters).toBeNull(); - } - }); - - test("allows parameters: undefined", () => { - const result = evaluatorDefinitionSchema.safeParse({ - parameters: undefined, - }); - expect(result.success).toBe(true); - if (result.success) { - expect(result.data.parameters).toBeUndefined(); - } - }); - - test("allows parameters omitted entirely", () => { - const result = evaluatorDefinitionSchema.safeParse({}); - expect(result.success).toBe(true); - if (result.success) { - expect(result.data.parameters).toBeUndefined(); - } - }); -}); diff --git a/js/src/zod/zod-v4-serialization.test.ts b/js/src/zod/zod-v4-serialization.test.ts deleted file mode 100644 index 9ba907e01..000000000 --- a/js/src/zod/zod-v4-serialization.test.ts +++ /dev/null @@ -1,141 +0,0 @@ -/** - * Tests for zod v4 compatibility with serializeEvalParametersToStaticParametersSchema - * - * This file tests makeEvalParametersSchema specifically with zod v4. - * See zod-v3-serialization.test.ts for zod v3 specific tests. - */ - -import { test, describe, beforeEach, expect } from "vitest"; -import * as zodModule from "zod"; -import { z } from "zod"; -import { - EXPECTED_STRING_SCHEMA, - EXPECTED_NUMBER_SCHEMA, - EXPECTED_OBJECT_SCHEMA, - EXPECTED_ENUM_SCHEMA, - EXPECTED_ARRAY_SCHEMA, -} from "./zod-serialization-test-shared"; -import { serializeEvalParametersToStaticParametersSchema } from "../framework2"; -// Detect which zod version is installed by checking for v4-specific properties -function getInstalledZodVersion(): 3 | 4 { - const testSchema = zodModule.z.string(); - // eslint-disable-next-line @typescript-eslint/no-explicit-any - return "_zod" in (testSchema as any) ? 4 : 3; -} -describe("makeEvalParametersSchema with Zod v4", () => { - beforeEach(() => { - const version = getInstalledZodVersion(); - expect(version).toBe(4); - }); - - test("string schema serializes correctly", () => { - const parameters = { - instructions: z - .string() - .describe("The instructions for the agent") - .default("You are a helpful assistant."), - }; - - const result = serializeEvalParametersToStaticParametersSchema(parameters); - expect(result.instructions).toBeDefined(); - expect(result.instructions.type).toBe("data"); - expect( - result.instructions.type === "data" && result.instructions.schema, - ).toStrictEqual(EXPECTED_STRING_SCHEMA); - expect(result.instructions.description).toBe( - "The instructions for the agent", - ); - expect(result.instructions.default).toBe("You are a helpful assistant."); - }); - - test("number schema serializes correctly", () => { - const parameters = { - temperature: z - .number() - .min(0) - .max(2) - .describe("Temperature for LLM") - .default(0.7), - }; - - const result = serializeEvalParametersToStaticParametersSchema(parameters); - - expect(result.temperature.type).toBe("data"); - expect( - result.temperature.type === "data" && result.temperature.schema, - ).toStrictEqual(EXPECTED_NUMBER_SCHEMA); - expect(result.temperature.description).toBe("Temperature for LLM"); - expect(result.temperature.default).toBe(0.7); - }); - - test("object schema serializes correctly", () => { - const parameters = { - config: z - .object({ - model: z.string(), - maxTokens: z.number().optional(), - }) - .describe("Configuration object"), - }; - - const result = serializeEvalParametersToStaticParametersSchema(parameters); - - expect(result.config.type).toBe("data"); - expect(result.config.type === "data" && result.config.schema).toStrictEqual( - EXPECTED_OBJECT_SCHEMA, - ); - expect(result.config.description).toBe("Configuration object"); - }); - - test("enum schema serializes correctly", () => { - const parameters = { - mode: z - .enum(["fast", "accurate", "balanced"]) - .describe("Processing mode") - .default("balanced"), - }; - - const result = serializeEvalParametersToStaticParametersSchema(parameters); - - expect(result.mode.type).toBe("data"); - expect(result.mode.type === "data" && result.mode.schema).toStrictEqual( - EXPECTED_ENUM_SCHEMA, - ); - expect(result.mode.description).toBe("Processing mode"); - expect(result.mode.default).toBe("balanced"); - }); - - test("array schema serializes correctly", () => { - const parameters = { - tags: z - .array(z.string()) - .describe("Tags for filtering") - .default(["default"]), - }; - - const result = serializeEvalParametersToStaticParametersSchema(parameters); - - expect(result.tags.type).toBe("data"); - expect(result.tags.type === "data" && result.tags.schema).toStrictEqual( - EXPECTED_ARRAY_SCHEMA, - ); - expect(result.tags.description).toBe("Tags for filtering"); - expect(result.tags.default).toEqual(["default"]); - }); - - test("model parameter serializes correctly", () => { - const parameters = { - model: { - type: "model" as const, - description: "Model to use", - default: "gpt-5-mini", - }, - }; - - const result = serializeEvalParametersToStaticParametersSchema(parameters); - - expect(result.model.type).toBe("model"); - expect(result.model.description).toBe("Model to use"); - expect(result.model.default).toBe("gpt-5-mini"); - }); -}); diff --git a/js/tests/api-compatibility/api-compatibility.test.ts b/js/tests/api-compatibility/api-compatibility.test.ts index 99768dd48..f11e3d847 100644 --- a/js/tests/api-compatibility/api-compatibility.test.ts +++ b/js/tests/api-compatibility/api-compatibility.test.ts @@ -20,7 +20,7 @@ import type { Options } from "tsup"; * ## How It Works * * 1. Downloads the latest published version from npm - * 2. Extracts the .d.ts files for each entrypoint (main, browser, dev, util) + * 2. Extracts the .d.ts files for each entrypoint (main, browser, util) * 3. Parses both published and current .d.ts files using TypeScript Compiler API * 4. Compares exported symbols (functions, classes, interfaces, types, etc.) * 5. Fails if breaking changes are detected in non-major version bumps @@ -100,7 +100,6 @@ function getEntrypointName(entryFile: string, outDir: string): string { if (entryFile.includes("src/browser/index.ts")) return "browser"; if (entryFile.includes("src/edge-light/index.ts")) return "edge-light"; if (entryFile.includes("src/workerd/index.ts")) return "workerd"; - if (entryFile.includes("dev/index.ts")) return "dev"; if (entryFile.includes("util/index.ts")) return "util"; // Default to basename @@ -152,6 +151,45 @@ function getVersionBumpType( return "none"; } +function getChangesetBumpType( + packageName: string, +): "major" | "minor" | "patch" | null { + const changesetDir = path.join(__dirname, "..", "..", "..", ".changeset"); + if (!fs.existsSync(changesetDir)) { + return null; + } + + const bumpPriority = { patch: 1, minor: 2, major: 3 } as const; + let highestBump: "major" | "minor" | "patch" | null = null; + const escapedPackageName = packageName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + const packageBumpPattern = new RegExp( + `^\\s*["']?${escapedPackageName}["']?\\s*:\\s*(major|minor|patch)\\s*$`, + "m", + ); + + for (const file of fs.readdirSync(changesetDir)) { + if (!file.endsWith(".md")) { + continue; + } + + const contents = fs.readFileSync(path.join(changesetDir, file), "utf8"); + const frontmatter = /^---\r?\n([\s\S]*?)\r?\n---/.exec(contents)?.[1]; + const bump = frontmatter?.match(packageBumpPattern)?.[1] as + | "major" + | "minor" + | "patch" + | undefined; + if ( + bump && + (!highestBump || bumpPriority[bump] > bumpPriority[highestBump]) + ) { + highestBump = bump; + } + } + + return highestBump; +} + /** * Extracts exports and their signatures from a .d.ts file. * For bulk exports (export { A, B, C }), looks up the actual declaration @@ -2857,13 +2895,21 @@ describe("API Compatibility", () => { // Determine version bump type const publishedVersionInfo = parseVersion(publishedVersion); const currentVersionInfo = parseVersion(currentVersion); - versionBumpType = getVersionBumpType( + const packageVersionBumpType = getVersionBumpType( publishedVersionInfo, currentVersionInfo, ); + const changesetBumpType = getChangesetBumpType("braintrust"); + const bumpPriority = { none: 0, patch: 1, minor: 2, major: 3 } as const; + versionBumpType = + changesetBumpType && + bumpPriority[changesetBumpType] > bumpPriority[packageVersionBumpType] + ? changesetBumpType + : packageVersionBumpType; console.log(`Published version: ${publishedVersion}`); console.log(`Current version: ${currentVersion}`); + console.log(`Changeset bump type: ${changesetBumpType ?? "none"}`); console.log(`Version bump type: ${versionBumpType}`); // Create temp directory for downloaded package diff --git a/js/tsup.config.ts b/js/tsup.config.ts index 14bca6b36..416ef9679 100644 --- a/js/tsup.config.ts +++ b/js/tsup.config.ts @@ -32,33 +32,6 @@ export default defineConfig([ define, clean: true, }, - { - entry: { cli: "src/cli/index.ts" }, - format: ["cjs"], - removeNodeProtocol: false, - outDir: "dist", - external: ["esbuild", "prettier", "typescript", "zod"], - // CLI doesn't need DTS - dts: false, - define, - clean: false, - }, - { - entry: ["dev/index.ts"], - format: ["cjs", "esm"], - outDir: "dev/dist", - removeNodeProtocol: false, - external: ["esbuild", "prettier", "typescript", "zod"], - dts: { - // Split DTS generation to reduce memory usage - compilerOptions: { - skipLibCheck: true, - }, - }, - splitting: true, - define, - clean: true, - }, { entry: ["util/index.ts"], format: ["cjs", "esm"], diff --git a/knip.jsonc b/knip.jsonc index 40f840f2d..202457490 100644 --- a/knip.jsonc +++ b/knip.jsonc @@ -55,7 +55,6 @@ "scripts/**/*.ts", "examples/**/*.{ts,tsx,js,jsx,mjs,cjs}", "smoke/**/*.{ts,tsx,js,jsx,mjs,cjs}", - "dev/**/*.ts", "util/**/*.ts", ], "project": [ @@ -64,7 +63,6 @@ "scripts/**/*.ts", "examples/**/*.{ts,tsx,js,jsx,mjs,cjs}", "smoke/**/*.{ts,tsx,js,jsx,mjs,cjs,d.ts}", - "dev/**/*.ts", "util/**/*.ts", ], }, diff --git a/package.json b/package.json index 78f725852..aa3b53f88 100644 --- a/package.json +++ b/package.json @@ -22,8 +22,6 @@ "dotagents:sync": "dotagents sync", "changeset": "changeset", "playground": "turbo run playground --filter=\"braintrust\"", - "playground:cli:push": "turbo run playground:cli:push --filter=\"braintrust\"", - "playground:cli:eval": "turbo run playground:cli:eval --filter=\"braintrust\"", "formatting": "prettier --check .", "lint": "turbo run lint", "fix:formatting": "prettier --write .", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 79b23cd1e..c6deed80a 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -316,9 +316,6 @@ importers: js: dependencies: - '@next/env': - specifier: ^14.2.3 - version: 14.2.3 '@vercel/functions': specifier: ^1.0.2 version: 1.0.2 @@ -331,24 +328,12 @@ importers: ajv: specifier: ^8.20.0 version: 8.20.0 - argparse: - specifier: ^2.0.1 - version: 2.0.1 astring: specifier: ^1.9.0 version: 1.9.0 cjs-module-lexer: specifier: ^2.2.0 version: 2.2.0 - cli-progress: - specifier: ^3.12.0 - version: 3.12.0 - cli-table3: - specifier: ^0.6.5 - version: 0.6.5 - cors: - specifier: ^2.8.5 - version: 2.8.5 dotenv: specifier: ^16.4.5 version: 16.4.5 @@ -361,36 +346,21 @@ importers: eventsource-parser: specifier: ^1.1.2 version: 1.1.2 - express: - specifier: ^5.2.1 - version: 5.2.1 - http-errors: - specifier: ^2.0.0 - version: 2.0.1 meriyah: specifier: ^6.1.4 version: 6.1.4 - minimatch: - specifier: ^10.2.5 - version: 10.2.5 module-details-from-path: specifier: ^1.0.4 version: 1.0.4 mustache: specifier: ^4.2.0 version: 4.2.0 - pluralize: - specifier: ^8.0.0 - version: 8.0.0 semifies: specifier: ^1.0.0 version: 1.0.0 source-map: specifier: ^0.7.4 version: 0.7.6 - termi-link: - specifier: ^1.0.1 - version: 1.1.0 unplugin: specifier: ^2.3.5 version: 2.3.11 @@ -404,39 +374,18 @@ importers: '@anthropic-ai/sdk': specifier: ^0.60.0 version: 0.60.0 - '@nodelib/fs.walk': - specifier: ^1.2.8 - version: 1.2.8 - '@types/argparse': - specifier: ^2.0.14 - version: 2.0.14 '@types/async': specifier: ^3.2.24 version: 3.2.24 - '@types/cli-progress': - specifier: ^3.11.5 - version: 3.11.5 - '@types/cors': - specifier: ^2.8.17 - version: 2.8.17 '@types/esquery': specifier: ^1.5.4 version: 1.5.4 - '@types/express': - specifier: ^5.0.0 - version: 5.0.1 - '@types/http-errors': - specifier: ^2.0.4 - version: 2.0.4 '@types/mustache': specifier: ^4.2.5 version: 4.2.5 '@types/node': specifier: ^20.10.5 version: 20.19.40 - '@types/pluralize': - specifier: ^0.0.30 - version: 0.0.30 '@typescript-eslint/eslint-plugin': specifier: ^8.49.0 version: 8.50.0(@typescript-eslint/parser@8.50.0(eslint@9.39.4(jiti@2.6.1))(typescript@5.4.4))(eslint@9.39.4(jiti@2.6.1))(typescript@5.4.4) @@ -686,10 +635,6 @@ packages: resolution: {integrity: sha512-EYlRokl8szrP9Z25qT5aepMdBjzBvHF9ZEhzIiUBc9guz/T31EqRgvD0QSgZcpE93xiwrr+OkB4nz0BZyF6fSA==} engines: {node: '>= 20.12.0'} - '@colors/colors@1.5.0': - resolution: {integrity: sha512-ooWCrlZP11i8GImSjTHYHLkvFDP48nS4+204nGb1RiX/WXYHmJA2III9/e2DWVabCESdW7hBAEzHRqUn9OUVvQ==} - engines: {node: '>=0.1.90'} - '@emnapi/core@1.11.1': resolution: {integrity: sha512-RSvbQmHzdKzNsLYa/wHrbc3KN4sYLKAdPZxqiM2HATqv/SBk2/ENSHpvXGaLOMcsAyz0poEGqkmmKYG3OWiJEQ==} @@ -1111,9 +1056,6 @@ packages: '@emnapi/core': ^1.7.1 '@emnapi/runtime': ^1.7.1 - '@next/env@14.2.3': - resolution: {integrity: sha512-W7fd7IbkfmeeY2gXrzJYDx8D2lWKbVoTIj1o1ScPHNzvp30s1AuoEFSdr39bC5sjxJaxTtq3OTCZboNp0lNWHA==} - '@nodelib/fs.scandir@2.1.5': resolution: {integrity: sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g==} engines: {node: '>= 8'} @@ -1779,27 +1721,12 @@ packages: '@tybys/wasm-util@0.10.3': resolution: {integrity: sha512-F3fo1MYrRJYL3zER0OUOmkutjr1Vp23m7OsSgp7nq4SP6OqX6C/56XFIPAl5bt3zaBRjmW7SGz3u/6LwFpYcOg==} - '@types/argparse@2.0.14': - resolution: {integrity: sha512-jJ6NMs9rXQ0rsqNt3TL4Elcwhd6wygo3lJOVoiHzURD34vsCcAlw443uGu4PXTtEmMF7sYKoadTCLXNmuJuQGw==} - '@types/async@3.2.24': resolution: {integrity: sha512-8iHVLHsCCOBKjCF2KwFe0p9Z3rfM9mL+sSP8btyR5vTjJRAqpBYD28/ZLgXPf0pjG1VxOvtCV/BgXkQbpSe8Hw==} - '@types/body-parser@1.19.5': - resolution: {integrity: sha512-fB3Zu92ucau0iQ0JMCFQE7b/dv8Ot07NI3KaZIkIUNXq82k4eBAqUaneXfleGY9JWskeS9y+u0nXMyspcuQrCg==} - '@types/chai@5.2.3': resolution: {integrity: sha512-Mw558oeA9fFbv65/y4mHtXDs9bPnFMZAL/jxdPFUpOHHIXX91mcgEHbS5Lahr+pwZFR8A7GQleRWeI6cGFC2UA==} - '@types/cli-progress@3.11.5': - resolution: {integrity: sha512-D4PbNRbviKyppS5ivBGyFO29POlySLmA2HyUFE4p5QGazAMM3CwkKWcvTl8gvElSuxRh6FPKL8XmidX873ou4g==} - - '@types/connect@3.4.38': - resolution: {integrity: sha512-K6uROf1LD88uDQqJCktA4yzL1YYAK6NgfsI0v/mTgyPKWsX1CnJ0XPSDhViejru1GcRkLWb8RlzFYJRqGUbaug==} - - '@types/cors@2.8.17': - resolution: {integrity: sha512-8CGDvrBj1zgo2qE+oS3pOCyYNqCPryMWY2bGfwA0dcfopWGgxs+78df0Rs3rc9THP4JkOhLsAa+15VdpAqkcUA==} - '@types/deep-eql@4.0.2': resolution: {integrity: sha512-c9h9dVVMigMPc4bwTvC5dxqtqJZwQPePsWjPlpSOnojbor6pGqdk541lfA7AqFQr5pB1BRdq0juY9db81BwyFw==} @@ -1818,21 +1745,9 @@ packages: '@types/estree@1.0.9': resolution: {integrity: sha512-GhdPgy1el4/ImP05X05Uw4cw2/M93BCUmnEvWZNStlCzEKME4Fkk+YpoA5OiHNQmoS7Cafb8Xa3Pya8m1Qrzeg==} - '@types/express-serve-static-core@5.0.6': - resolution: {integrity: sha512-3xhRnjJPkULekpSzgtoNYYcTWgEZkp4myc+Saevii5JPnHNvHMRlBSHDbs7Bh1iPPoVTERHEZXyhyLbMEsExsA==} - - '@types/express@5.0.1': - resolution: {integrity: sha512-UZUw8vjpWFXuDnjFTh7/5c2TWDlQqeXHi6hcN7F2XSVT5P+WmUnnbFS3KA6Jnc6IsEqI2qCVu2bK0R0J4A8ZQQ==} - - '@types/http-errors@2.0.4': - resolution: {integrity: sha512-D0CFMMtydbJAegzOyHjtiKPLlvnm3iTZyZRSZoLq2mRhDdmLfIWOCYPfQJ4cu2erKghU++QvjcUjp/5h7hESpA==} - '@types/json-schema@7.0.15': resolution: {integrity: sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA==} - '@types/mime@1.3.5': - resolution: {integrity: sha512-/pyBZWSLD2n0dcHE3hq8s8ZvcETHtEuF+3E7XVt0Ig2nvsVQXdghHVcEkIWjy9A0wKfTn97a/PSDYohKIlnP/w==} - '@types/mustache@4.2.5': resolution: {integrity: sha512-PLwiVvTBg59tGFL/8VpcGvqOu3L4OuveNvPi0EYbWchRdEVP++yRUXJPFl+CApKEq13017/4Nf7aQ5lTtHUNsA==} @@ -1851,24 +1766,9 @@ packages: '@types/nunjucks@3.2.6': resolution: {integrity: sha512-pHiGtf83na1nCzliuAdq8GowYiXvH5l931xZ0YEHaLMNFgynpEqx+IPStlu7UaDkehfvl01e4x/9Tpwhy7Ue3w==} - '@types/pluralize@0.0.30': - resolution: {integrity: sha512-kVww6xZrW/db5BR9OqiT71J9huRdQ+z/r+LbDuT7/EK50mCmj5FoaIARnVv0rvjUS/YpDox0cDU9lpQT011VBA==} - - '@types/qs@6.9.18': - resolution: {integrity: sha512-kK7dgTYDyGqS+e2Q4aK9X3D7q234CIZ1Bv0q/7Z5IwRDoADNU81xXJK/YVyLbLTZCoIwUoDoffFeF+p/eIklAA==} - - '@types/range-parser@1.2.7': - resolution: {integrity: sha512-hKormJbkJqzQGhziax5PItDUTMAM9uE2XXQmM37dyd4hVM+5aVl7oVxMVUiVQn2oCQFN/LKCZdvSM0pFRqbSmQ==} - '@types/retry@0.12.0': resolution: {integrity: sha512-wWKOClTTiizcZhXnPY4wikVAwmdYHp8q6DmC+EJUzAMsycb7HB32Kh9RN4+0gExjmPmZSAQjgURXIGATPegAvA==} - '@types/send@0.17.4': - resolution: {integrity: sha512-x2EM6TJOybec7c52BX0ZspPodMsQUd5L6PRwOunVyVUhXiBSKf3AezDL8Dgvgt5o0UfKNfuA0eMLr2wLT4AiBA==} - - '@types/serve-static@1.15.7': - resolution: {integrity: sha512-W8Ym+h8nhuRwaKPaDw34QUkwsGi6Rc4yYqvKFo5rm2FUEhCFbzVWrxXUxuKK8TASjWsysJY0nsmNCGhCOIsrOw==} - '@types/set-cookie-parser@2.4.10': resolution: {integrity: sha512-GGmQVGpQWUe5qglJozEjZV/5dyxbOOZ0LHe/lqyWssB88Y4svNfst0uqBVscdDeIKl5Jy5+aPSvy7mI9tYRguw==} @@ -2154,10 +2054,6 @@ packages: balanced-match@1.0.2: resolution: {integrity: sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==} - balanced-match@4.0.4: - resolution: {integrity: sha512-BLrgEcRTwX2o6gGxGOCNyMvGSp35YofuYzw9h1IMTRmKqttAZZVU67bdb9Pr2vUHA8+j3i2tJfjO6C6+4myGTA==} - engines: {node: 18 || 20 || >=22} - base64-js@1.5.1: resolution: {integrity: sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==} @@ -2179,10 +2075,6 @@ packages: brace-expansion@2.1.4: resolution: {integrity: sha512-hGfVzPxthbf3+2yjg/RBs60cB0FhqBS/zvdV/4wn4/BmN0bNMMHPc4V/BbFieqf1TKAGGAHnY4eSjajCl0f2Xg==} - brace-expansion@5.0.9: - resolution: {integrity: sha512-ScQ4IuvIEF1TMlP7Zt+vjJ//9zlPb2SDcxWxM3bk8s6t6GGdJ7KO1dCcTidOPJKePW30LE/2cT7wCyPho9/Wxg==} - engines: {node: 20 || >=22} - braces@3.0.3: resolution: {integrity: sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==} engines: {node: '>=8'} @@ -2262,14 +2154,6 @@ packages: resolution: {integrity: sha512-aCj4O5wKyszjMmDT4tZj93kxyydN/K5zPWSCe6/0AV/AA1pqe5ZBIw0a2ZfPQV7lL5/yb5HsUreJ6UFAF1tEQw==} engines: {node: '>=18'} - cli-progress@3.12.0: - resolution: {integrity: sha512-tRkV3HJ1ASwm19THiiLIXLO7Im7wlTuKnvkYaTkyoAPefqjNg7W7DHKUlGRxy9vxDvbyCYQkQozvptuMkGCg8A==} - engines: {node: '>=4'} - - cli-table3@0.6.5: - resolution: {integrity: sha512-+W/5efTR7y5HRD7gACw9yQjqMVvEMLBHmboM/kPWam+H+Hmyrgjh6YncVKK122YZkXrLudzTuAukUw9FnMf7IQ==} - engines: {node: 10.* || >= 12.*} - cli-truncate@5.1.1: resolution: {integrity: sha512-SroPvNHxUnk+vIW/dOSfNqdy1sPEFkrTk6TUtqLCnBlo3N7TNYYkzzN7uSD6+jVjrdO4+p8nH7JzH6cIvUem6A==} engines: {node: '>=20'} @@ -3201,10 +3085,6 @@ packages: resolution: {integrity: sha512-VP79XUPxV2CigYP3jWwAUFSku2aKqBH7uTAapFWCBqutsbmDo96KY5o8uh6U+/YSIn5OxJnXp73beVkpqMIGhA==} engines: {node: '>=18'} - minimatch@10.2.5: - resolution: {integrity: sha512-MULkVLfKGYDFYejP07QOurDLLQpcjk7Fw+7jXS2R2czRQzR56yHRveU5NDJEOviH+hETZKSkIk5c+T23GjFUMg==} - engines: {node: 18 || 20 || >=22} - minimatch@3.1.5: resolution: {integrity: sha512-VgjWUsnnT6n+NUk6eZq77zeFdpW2LWDzP6zFGrCbHXiYNul5Dzqk2HHQ5uFH2DNW5Xbp8+jVzaeNt94ssEEl4w==} @@ -3506,10 +3386,6 @@ packages: pkg-types@1.3.1: resolution: {integrity: sha512-/Jm5M4RvtBFVkKWRu2BLUTNP8/M2a+UwuAX+ae4770q1qVGtfjG+WTCupoZixokjmHiry8uI+dlY8KXYV5HVVQ==} - pluralize@8.0.0: - resolution: {integrity: sha512-Nc3IT5yHzflTfbjgqWcCPpo7DaKy4FnpB0l/zCAW0Tc7jxAiuqSxHasntB3D7887LSrA93kDJ9IXovxJYxyLCA==} - engines: {node: '>=4'} - postcss-load-config@6.0.1: resolution: {integrity: sha512-oPtTM4oerL+UXmx+93ytZVN82RrlY/wPUV8IeDxFrzIjXOLF1pN+EmKPLbubvKHT2HC20xXsCAH2Z+CKV6Oz/g==} engines: {node: '>= 18'} @@ -3853,10 +3729,6 @@ packages: resolution: {integrity: sha512-wK0Ri4fOGjv/XPy8SBHZChl8CM7uMc5VML7SqiQ0zG7+J5Vr+RMQDoHa2CNT6KHUnTGIXH34UDMkPzAUyapBZg==} engines: {node: '>=8'} - termi-link@1.1.0: - resolution: {integrity: sha512-2qSN6TnomHgVLtk+htSWbaYs4Rd2MH/RU7VpHTy6MBstyNyWbM4yKd1DCYpE3fDg8dmGWojXCngNi/MHCzGuAA==} - engines: {node: '>=12'} - terser-webpack-plugin@5.6.0: resolution: {integrity: sha512-Eum+5ajkaOhf5KbM26osvv21kLD7BaGqQ1UA4Ami4arYwylmGUQTgHFpHDdmJod1q4QXa66p0to/FBKID+J1vA==} engines: {node: '>= 10.13.0'} @@ -4558,9 +4430,6 @@ snapshots: fast-wrap-ansi: 0.2.0 sisteransi: 1.0.5 - '@colors/colors@1.5.0': - optional: true - '@emnapi/core@1.11.1': dependencies: '@emnapi/wasi-threads': 1.2.2 @@ -4986,8 +4855,6 @@ snapshots: '@tybys/wasm-util': 0.10.3 optional: true - '@next/env@14.2.3': {} - '@nodelib/fs.scandir@2.1.5': dependencies: '@nodelib/fs.stat': 2.0.5 @@ -5548,32 +5415,13 @@ snapshots: tslib: 2.8.1 optional: true - '@types/argparse@2.0.14': {} - '@types/async@3.2.24': {} - '@types/body-parser@1.19.5': - dependencies: - '@types/connect': 3.4.38 - '@types/node': 22.19.1 - '@types/chai@5.2.3': dependencies: '@types/deep-eql': 4.0.2 assertion-error: 2.0.1 - '@types/cli-progress@3.11.5': - dependencies: - '@types/node': 22.19.1 - - '@types/connect@3.4.38': - dependencies: - '@types/node': 22.19.1 - - '@types/cors@2.8.17': - dependencies: - '@types/node': 22.19.1 - '@types/deep-eql@4.0.2': {} '@types/eslint-scope@3.7.7': @@ -5594,25 +5442,8 @@ snapshots: '@types/estree@1.0.9': {} - '@types/express-serve-static-core@5.0.6': - dependencies: - '@types/node': 22.19.1 - '@types/qs': 6.9.18 - '@types/range-parser': 1.2.7 - '@types/send': 0.17.4 - - '@types/express@5.0.1': - dependencies: - '@types/body-parser': 1.19.5 - '@types/express-serve-static-core': 5.0.6 - '@types/serve-static': 1.15.7 - - '@types/http-errors@2.0.4': {} - '@types/json-schema@7.0.15': {} - '@types/mime@1.3.5': {} - '@types/mustache@4.2.5': {} '@types/node@12.20.55': {} @@ -5631,25 +5462,8 @@ snapshots: '@types/nunjucks@3.2.6': {} - '@types/pluralize@0.0.30': {} - - '@types/qs@6.9.18': {} - - '@types/range-parser@1.2.7': {} - '@types/retry@0.12.0': {} - '@types/send@0.17.4': - dependencies: - '@types/mime': 1.3.5 - '@types/node': 22.19.1 - - '@types/serve-static@1.15.7': - dependencies: - '@types/http-errors': 2.0.4 - '@types/node': 22.19.1 - '@types/send': 0.17.4 - '@types/set-cookie-parser@2.4.10': dependencies: '@types/node': 22.19.1 @@ -5914,6 +5728,7 @@ snapshots: dependencies: mime-types: 3.0.1 negotiator: 1.0.0 + optional: true acorn-import-attributes@1.9.5(acorn@8.16.0): dependencies: @@ -6007,8 +5822,6 @@ snapshots: balanced-match@1.0.2: {} - balanced-match@4.0.4: {} - base64-js@1.5.1: {} baseline-browser-mapping@2.9.14: {} @@ -6030,6 +5843,7 @@ snapshots: type-is: 2.1.0 transitivePeerDependencies: - supports-color + optional: true brace-expansion@1.1.18: dependencies: @@ -6040,10 +5854,6 @@ snapshots: dependencies: balanced-match: 1.0.2 - brace-expansion@5.0.9: - dependencies: - balanced-match: 4.0.4 - braces@3.0.3: dependencies: fill-range: 7.1.1 @@ -6063,7 +5873,8 @@ snapshots: esbuild: 0.28.1 load-tsconfig: 0.2.5 - bytes@3.1.2: {} + bytes@3.1.2: + optional: true cac@6.7.14: {} @@ -6071,11 +5882,13 @@ snapshots: dependencies: es-errors: 1.3.0 function-bind: 1.1.2 + optional: true call-bound@1.0.4: dependencies: call-bind-apply-helpers: 1.0.2 get-intrinsic: 1.3.0 + optional: true callsites@3.1.0: {} @@ -6108,16 +5921,6 @@ snapshots: dependencies: restore-cursor: 5.1.0 - cli-progress@3.12.0: - dependencies: - string-width: 4.2.3 - - cli-table3@0.6.5: - dependencies: - string-width: 4.2.3 - optionalDependencies: - '@colors/colors': 1.5.0 - cli-truncate@5.1.1: dependencies: slice-ansi: 7.1.2 @@ -6162,16 +5965,21 @@ snapshots: content-disposition@1.0.0: dependencies: safe-buffer: 5.2.1 + optional: true - content-type@1.0.5: {} + content-type@1.0.5: + optional: true - content-type@2.0.0: {} + content-type@2.0.0: + optional: true convert-source-map@2.0.0: {} - cookie-signature@1.2.2: {} + cookie-signature@1.2.2: + optional: true - cookie@0.7.2: {} + cookie@0.7.2: + optional: true cookie@1.1.1: {} @@ -6179,6 +5987,7 @@ snapshots: dependencies: object-assign: 4.1.1 vary: 1.1.2 + optional: true cross-env@7.0.3: dependencies: @@ -6200,7 +6009,8 @@ snapshots: deep-is@0.1.4: {} - depd@2.0.0: {} + depd@2.0.0: + optional: true detect-indent@6.1.0: {} @@ -6219,10 +6029,12 @@ snapshots: call-bind-apply-helpers: 1.0.2 es-errors: 1.3.0 gopd: 1.2.0 + optional: true eastasianwidth@0.2.0: {} - ee-first@1.1.1: {} + ee-first@1.1.1: + optional: true electron-to-chromium@1.5.267: {} @@ -6232,7 +6044,8 @@ snapshots: emoji-regex@9.2.2: {} - encodeurl@2.0.0: {} + encodeurl@2.0.0: + optional: true encoding@0.1.13: dependencies: @@ -6251,15 +6064,18 @@ snapshots: environment@1.1.0: {} - es-define-property@1.0.1: {} + es-define-property@1.0.1: + optional: true - es-errors@1.3.0: {} + es-errors@1.3.0: + optional: true es-module-lexer@2.1.0: {} es-object-atoms@1.1.1: dependencies: es-errors: 1.3.0 + optional: true esbuild@0.28.1: optionalDependencies: @@ -6292,7 +6108,8 @@ snapshots: escalade@3.2.0: {} - escape-html@1.0.3: {} + escape-html@1.0.3: + optional: true escape-string-regexp@4.0.0: {} @@ -6420,7 +6237,8 @@ snapshots: esutils@2.0.3: {} - etag@1.8.1: {} + etag@1.8.1: + optional: true event-target-shim@5.0.1: {} @@ -6479,6 +6297,7 @@ snapshots: vary: 1.1.2 transitivePeerDependencies: - supports-color + optional: true extendable-error@0.1.7: {} @@ -6538,6 +6357,7 @@ snapshots: statuses: 2.0.2 transitivePeerDependencies: - supports-color + optional: true find-up@4.1.0: dependencies: @@ -6571,9 +6391,11 @@ snapshots: dependencies: fd-package-json: 2.0.0 - forwarded@0.2.0: {} + forwarded@0.2.0: + optional: true - fresh@2.0.0: {} + fresh@2.0.0: + optional: true fs-extra@7.0.1: dependencies: @@ -6594,7 +6416,8 @@ snapshots: fsevents@2.3.3: optional: true - function-bind@1.1.2: {} + function-bind@1.1.2: + optional: true get-caller-file@2.0.5: {} @@ -6612,11 +6435,13 @@ snapshots: has-symbols: 1.1.0 hasown: 2.0.3 math-intrinsics: 1.1.0 + optional: true get-proto@1.0.1: dependencies: dunder-proto: 1.0.1 es-object-atoms: 1.1.1 + optional: true get-tsconfig@4.13.0: dependencies: @@ -6669,7 +6494,8 @@ snapshots: globrex@0.1.2: {} - gopd@1.2.0: {} + gopd@1.2.0: + optional: true graceful-fs@4.2.11: {} @@ -6686,11 +6512,13 @@ snapshots: has-flag@4.0.0: {} - has-symbols@1.1.0: {} + has-symbols@1.1.0: + optional: true hasown@2.0.3: dependencies: function-bind: 1.1.2 + optional: true headers-polyfill@5.0.1: dependencies: @@ -6709,6 +6537,7 @@ snapshots: setprototypeof: 1.2.0 statuses: 2.0.2 toidentifier: 1.0.1 + optional: true human-id@4.1.3: {} @@ -6751,7 +6580,8 @@ snapshots: ip-address@10.4.0: optional: true - ipaddr.js@1.9.1: {} + ipaddr.js@1.9.1: + optional: true is-extglob@2.1.1: {} @@ -6769,7 +6599,8 @@ snapshots: is-number@7.0.0: {} - is-promise@4.0.0: {} + is-promise@4.0.0: + optional: true is-subdir@1.2.0: dependencies: @@ -6983,9 +6814,11 @@ snapshots: marked@4.3.0: {} - math-intrinsics@1.1.0: {} + math-intrinsics@1.1.0: + optional: true - media-typer@1.1.0: {} + media-typer@1.1.0: + optional: true memfs@4.51.1: dependencies: @@ -6996,7 +6829,8 @@ snapshots: tree-dump: 1.1.0(tslib@2.8.1) tslib: 2.8.1 - merge-descriptors@2.0.0: {} + merge-descriptors@2.0.0: + optional: true merge-stream@2.0.0: {} @@ -7014,13 +6848,10 @@ snapshots: mime-types@3.0.1: dependencies: mime-db: 1.54.0 + optional: true mimic-function@5.0.1: {} - minimatch@10.2.5: - dependencies: - brace-expansion: 5.0.9 - minimatch@3.1.5: dependencies: brace-expansion: 1.1.18 @@ -7171,7 +7002,8 @@ snapshots: natural-compare@1.4.0: {} - negotiator@1.0.0: {} + negotiator@1.0.0: + optional: true neo-async@2.6.2: {} @@ -7206,13 +7038,15 @@ snapshots: object-assign@4.1.1: {} - object-inspect@1.13.4: {} + object-inspect@1.13.4: + optional: true obug@2.1.1: {} on-finished@2.4.1: dependencies: ee-first: 1.1.1 + optional: true once@1.4.0: dependencies: @@ -7327,7 +7161,8 @@ snapshots: dependencies: callsites: 3.1.0 - parseurl@1.3.3: {} + parseurl@1.3.3: + optional: true path-exists@4.0.0: {} @@ -7342,7 +7177,8 @@ snapshots: path-to-regexp@6.3.0: {} - path-to-regexp@8.4.0: {} + path-to-regexp@8.4.0: + optional: true path-type@4.0.0: {} @@ -7369,8 +7205,6 @@ snapshots: mlly: 1.8.0 pathe: 2.0.3 - pluralize@8.0.0: {} - postcss-load-config@6.0.1(jiti@2.6.1)(postcss@8.5.19)(tsx@4.21.0)(yaml@2.9.0): dependencies: lilconfig: 3.1.3 @@ -7414,18 +7248,21 @@ snapshots: dependencies: forwarded: 0.2.0 ipaddr.js: 1.9.1 + optional: true punycode@2.3.1: {} qs@6.15.2: dependencies: side-channel: 1.1.0 + optional: true quansync@0.2.11: {} queue-microtask@1.2.3: {} - range-parser@1.2.1: {} + range-parser@1.2.1: + optional: true raw-body@3.0.2: dependencies: @@ -7433,6 +7270,7 @@ snapshots: http-errors: 2.0.1 iconv-lite: 0.7.2 unpipe: 1.0.0 + optional: true react-dom@19.2.4(react@19.2.4): dependencies: @@ -7536,6 +7374,7 @@ snapshots: path-to-regexp: 8.4.0 transitivePeerDependencies: - supports-color + optional: true run-parallel@1.2.0: dependencies: @@ -7545,7 +7384,8 @@ snapshots: dependencies: tslib: 2.8.1 - safe-buffer@5.2.1: {} + safe-buffer@5.2.1: + optional: true safer-buffer@2.1.2: {} @@ -7578,6 +7418,7 @@ snapshots: statuses: 2.0.2 transitivePeerDependencies: - supports-color + optional: true serve-static@2.2.0: dependencies: @@ -7587,10 +7428,12 @@ snapshots: send: 1.2.0 transitivePeerDependencies: - supports-color + optional: true set-cookie-parser@3.1.0: {} - setprototypeof@1.2.0: {} + setprototypeof@1.2.0: + optional: true shebang-command@2.0.0: dependencies: @@ -7609,6 +7452,7 @@ snapshots: dependencies: es-errors: 1.3.0 object-inspect: 1.13.4 + optional: true side-channel-map@1.0.1: dependencies: @@ -7616,6 +7460,7 @@ snapshots: es-errors: 1.3.0 get-intrinsic: 1.3.0 object-inspect: 1.13.4 + optional: true side-channel-weakmap@1.0.2: dependencies: @@ -7624,6 +7469,7 @@ snapshots: get-intrinsic: 1.3.0 object-inspect: 1.13.4 side-channel-map: 1.0.1 + optional: true side-channel@1.1.0: dependencies: @@ -7632,6 +7478,7 @@ snapshots: side-channel-list: 1.0.0 side-channel-map: 1.0.1 side-channel-weakmap: 1.0.2 + optional: true siginfo@2.0.0: {} @@ -7757,8 +7604,6 @@ snapshots: term-size@2.2.1: {} - termi-link@1.1.0: {} - terser-webpack-plugin@5.6.0(@swc/core@1.15.8)(esbuild@0.28.1)(postcss@8.5.19)(webpack@5.106.2(@swc/core@1.15.8)(esbuild@0.28.1)(postcss@8.5.19)): dependencies: '@jridgewell/trace-mapping': 0.3.31 @@ -7815,7 +7660,8 @@ snapshots: dependencies: is-number: 7.0.0 - toidentifier@1.0.1: {} + toidentifier@1.0.1: + optional: true tough-cookie@6.0.1: dependencies: @@ -7964,6 +7810,7 @@ snapshots: content-type: 2.0.0 media-typer: 1.1.0 mime-types: 3.0.1 + optional: true typedoc-plugin-markdown@3.17.1(typedoc@0.25.13(typescript@5.4.4)): dependencies: @@ -7999,7 +7846,8 @@ snapshots: universalify@0.1.2: {} - unpipe@1.0.0: {} + unpipe@1.0.0: + optional: true unplugin@2.3.11: dependencies: @@ -8022,7 +7870,8 @@ snapshots: uuid@11.1.1: {} - vary@1.1.2: {} + vary@1.1.2: + optional: true vite-tsconfig-paths@4.3.2(typescript@5.4.4)(vite@8.1.5(@types/node@20.19.40)(esbuild@0.28.1)(jiti@2.6.1)(terser@5.44.1)(tsx@4.21.0)(yaml@2.9.0)): dependencies: diff --git a/turbo.json b/turbo.json index 6e5da8d56..6afc75def 100644 --- a/turbo.json +++ b/turbo.json @@ -98,38 +98,6 @@ "HUGGINGFACE_API_KEY" ], "dependsOn": ["^build", "build"] - }, - "playground:cli:push": { - "cache": false, - "passThroughEnv": [ - "ANTHROPIC_API_KEY", - "BRAINTRUST_API_KEY", - "GEMINI_API_KEY", - "COHERE_API_KEY", - "CURSOR_API_KEY", - "GROQ_API_KEY", - "OPENAI_API_KEY", - "OPENROUTER_API_KEY", - "MISTRAL_API_KEY", - "HUGGINGFACE_API_KEY" - ], - "dependsOn": ["^build", "build"] - }, - "playground:cli:eval": { - "cache": false, - "passThroughEnv": [ - "ANTHROPIC_API_KEY", - "BRAINTRUST_API_KEY", - "GEMINI_API_KEY", - "COHERE_API_KEY", - "CURSOR_API_KEY", - "GROQ_API_KEY", - "OPENAI_API_KEY", - "OPENROUTER_API_KEY", - "MISTRAL_API_KEY", - "HUGGINGFACE_API_KEY" - ], - "dependsOn": ["^build", "build"] } } } From ac1fde4e1d010cc44428ecceccd7ffa9fd6b04e7 Mon Sep 17 00:00:00 2001 From: Luca Forstner Date: Thu, 20 Aug 2026 17:26:40 +0200 Subject: [PATCH 4/8] ref(eve)!: Bump minimum eve version to `0.34.0` (#2385) ref [braintrustdata/braintrust-sdk-javascript#1841](https://github.com/braintrustdata/braintrust-sdk-javascript/issues/1841) --- .changeset/remove-eve-pre-034.md | 5 + e2e/config/pr-comment-scenarios.json | 2 +- .../eve-v0-latest.span-tree.json | 12 +- .../__snapshots__/eve-v0-latest.span-tree.txt | 12 +- .../__snapshots__/eve-v0.span-tree.json | 12 +- .../__snapshots__/eve-v0.span-tree.txt | 12 +- .../eve-instrumentation/agent/agent.ts | 10 +- .../eve-instrumentation/agent/channels/eve.ts | 7 + .../agent/hooks/braintrust.ts | 15 - .../braintrust.ts} | 8 +- .../agent/subagents/researcher/agent.ts | 3 + .../subagents/researcher/hooks/braintrust.ts | 15 - .../researcher/instrumentation/braintrust.ts | 18 + .../eve-instrumentation/cassette-filter.mjs | 114 +- .../eve-instrumentation/package.json | 6 +- .../eve-instrumentation/pnpm-lock.yaml | 89 +- .../eve-instrumentation/scenario.test.ts | 158 +- e2e/scenarios/eve-instrumentation/scenario.ts | 25 +- js/src/exports.ts | 5 +- js/src/instrumentation/index.ts | 5 +- .../plugins/eve-plugin.test.ts | 2929 ++++++---------- js/src/instrumentation/plugins/eve-plugin.ts | 3005 ++++++----------- js/src/vendor-sdk-types/eve.ts | 608 +--- js/src/wrappers/attachment-utils.ts | 20 + 24 files changed, 2488 insertions(+), 4607 deletions(-) create mode 100644 .changeset/remove-eve-pre-034.md create mode 100644 e2e/scenarios/eve-instrumentation/agent/channels/eve.ts delete mode 100644 e2e/scenarios/eve-instrumentation/agent/hooks/braintrust.ts rename e2e/scenarios/eve-instrumentation/agent/{instrumentation.ts => instrumentation/braintrust.ts} (66%) delete mode 100644 e2e/scenarios/eve-instrumentation/agent/subagents/researcher/hooks/braintrust.ts create mode 100644 e2e/scenarios/eve-instrumentation/agent/subagents/researcher/instrumentation/braintrust.ts diff --git a/.changeset/remove-eve-pre-034.md b/.changeset/remove-eve-pre-034.md new file mode 100644 index 000000000..1e3034506 --- /dev/null +++ b/.changeset/remove-eve-pre-034.md @@ -0,0 +1,5 @@ +--- +"braintrust": major +--- + +ref(eve)!: Bump minimum eve version to `0.34.0` diff --git a/e2e/config/pr-comment-scenarios.json b/e2e/config/pr-comment-scenarios.json index 4399ab7e3..6db8e7692 100644 --- a/e2e/config/pr-comment-scenarios.json +++ b/e2e/config/pr-comment-scenarios.json @@ -623,7 +623,7 @@ "label": "Eve Instrumentation", "metadataScenario": "eve-instrumentation", "variants": [ - { "variantKey": "eve-v0", "label": "v0 pinned" }, + { "variantKey": "eve-v0", "label": "v0.34 pinned" }, { "variantKey": "eve-v0-latest", "label": "v0 latest" } ] }, diff --git a/e2e/scenarios/eve-instrumentation/__snapshots__/eve-v0-latest.span-tree.json b/e2e/scenarios/eve-instrumentation/__snapshots__/eve-v0-latest.span-tree.json index f9869502c..c3eedeee9 100644 --- a/e2e/scenarios/eve-instrumentation/__snapshots__/eve-v0-latest.span-tree.json +++ b/e2e/scenarios/eve-instrumentation/__snapshots__/eve-v0-latest.span-tree.json @@ -53,6 +53,8 @@ ], "metadata": { "eve.session_id": "", + "model": "qwen/qwen3-30b-a3b", + "provider": "openrouter", "scenario": "eve-instrumentation", "testRunId": "" }, @@ -233,8 +235,6 @@ "output": "Researcher result: [Eve instrumentation](https://eve.dev/docs/guides/instrumentation)", "metadata": { "eve.session_id": "", - "model": "qwen/qwen3-30b-a3b", - "provider": "openrouter", "scenario": "eve-instrumentation", "testRunId": "" }, @@ -365,6 +365,8 @@ ], "metadata": { "eve.session_id": "", + "model": "qwen/qwen3-30b-a3b", + "provider": "openrouter", "scenario": "eve-instrumentation", "testRunId": "" }, @@ -525,6 +527,8 @@ ], "metadata": { "eve.session_id": "", + "model": "qwen/qwen3-30b-a3b", + "provider": "openrouter", "scenario": "eve-instrumentation", "testRunId": "" }, @@ -705,8 +709,6 @@ "output": "Researcher result: Eve instrumentation https://eve.dev/docs/guides/instrumentation", "metadata": { "eve.session_id": "", - "model": "qwen/qwen3-30b-a3b", - "provider": "openrouter", "scenario": "eve-instrumentation", "testRunId": "" }, @@ -912,6 +914,8 @@ ], "metadata": { "eve.session_id": "", + "model": "qwen/qwen3-30b-a3b", + "provider": "openrouter", "scenario": "eve-instrumentation", "testRunId": "" }, diff --git a/e2e/scenarios/eve-instrumentation/__snapshots__/eve-v0-latest.span-tree.txt b/e2e/scenarios/eve-instrumentation/__snapshots__/eve-v0-latest.span-tree.txt index 1006edff0..feec9910f 100644 --- a/e2e/scenarios/eve-instrumentation/__snapshots__/eve-v0-latest.span-tree.txt +++ b/e2e/scenarios/eve-instrumentation/__snapshots__/eve-v0-latest.span-tree.txt @@ -65,6 +65,8 @@ span_tree: │ │ ] │ │ metadata: { │ │ "eve.session_id": "", +│ │ "model": "qwen/qwen3-30b-a3b", +│ │ "provider": "openrouter", │ │ "scenario": "eve-instrumentation", │ │ "testRunId": "" │ │ } @@ -95,8 +97,6 @@ span_tree: │ │ output: "Researcher result: [Eve instrumentation](https://eve.dev/docs/guides/instrumentation)" │ │ metadata: { │ │ "eve.session_id": "", -│ │ "model": "qwen/qwen3-30b-a3b", -│ │ "provider": "openrouter", │ │ "scenario": "eve-instrumentation", │ │ "testRunId": "" │ │ } @@ -347,6 +347,8 @@ span_tree: │ ] │ metadata: { │ "eve.session_id": "", +│ "model": "qwen/qwen3-30b-a3b", +│ "provider": "openrouter", │ "scenario": "eve-instrumentation", │ "testRunId": "" │ } @@ -498,6 +500,8 @@ span_tree: │ ] │ metadata: { │ "eve.session_id": "", + │ "model": "qwen/qwen3-30b-a3b", + │ "provider": "openrouter", │ "scenario": "eve-instrumentation", │ "testRunId": "" │ } @@ -528,8 +532,6 @@ span_tree: │ output: "Researcher result: Eve instrumentation https://eve.dev/docs/guides/instrumentation" │ metadata: { │ "eve.session_id": "", - │ "model": "qwen/qwen3-30b-a3b", - │ "provider": "openrouter", │ "scenario": "eve-instrumentation", │ "testRunId": "" │ } @@ -855,6 +857,8 @@ span_tree: ] metadata: { "eve.session_id": "", + "model": "qwen/qwen3-30b-a3b", + "provider": "openrouter", "scenario": "eve-instrumentation", "testRunId": "" } diff --git a/e2e/scenarios/eve-instrumentation/__snapshots__/eve-v0.span-tree.json b/e2e/scenarios/eve-instrumentation/__snapshots__/eve-v0.span-tree.json index 6066e92ae..af1d20932 100644 --- a/e2e/scenarios/eve-instrumentation/__snapshots__/eve-v0.span-tree.json +++ b/e2e/scenarios/eve-instrumentation/__snapshots__/eve-v0.span-tree.json @@ -53,6 +53,8 @@ ], "metadata": { "eve.session_id": "", + "model": "qwen/qwen3-30b-a3b", + "provider": "openrouter", "scenario": "eve-instrumentation", "testRunId": "" }, @@ -233,8 +235,6 @@ "output": "Researcher result: {\"title\": \"Eve instrumentation\", \"url\": \"https://eve.dev/docs/guides/instrumentation\"}", "metadata": { "eve.session_id": "", - "model": "qwen/qwen3-30b-a3b", - "provider": "openrouter", "scenario": "eve-instrumentation", "testRunId": "" }, @@ -365,6 +365,8 @@ ], "metadata": { "eve.session_id": "", + "model": "qwen/qwen3-30b-a3b", + "provider": "openrouter", "scenario": "eve-instrumentation", "testRunId": "" }, @@ -525,6 +527,8 @@ ], "metadata": { "eve.session_id": "", + "model": "qwen/qwen3-30b-a3b", + "provider": "openrouter", "scenario": "eve-instrumentation", "testRunId": "" }, @@ -705,8 +709,6 @@ "output": "Researcher result: Eve instrumentation (https://eve.dev/docs/guides/instrumentation)", "metadata": { "eve.session_id": "", - "model": "qwen/qwen3-30b-a3b", - "provider": "openrouter", "scenario": "eve-instrumentation", "testRunId": "" }, @@ -912,6 +914,8 @@ ], "metadata": { "eve.session_id": "", + "model": "qwen/qwen3-30b-a3b", + "provider": "openrouter", "scenario": "eve-instrumentation", "testRunId": "" }, diff --git a/e2e/scenarios/eve-instrumentation/__snapshots__/eve-v0.span-tree.txt b/e2e/scenarios/eve-instrumentation/__snapshots__/eve-v0.span-tree.txt index d86328682..1a17de807 100644 --- a/e2e/scenarios/eve-instrumentation/__snapshots__/eve-v0.span-tree.txt +++ b/e2e/scenarios/eve-instrumentation/__snapshots__/eve-v0.span-tree.txt @@ -65,6 +65,8 @@ span_tree: │ │ ] │ │ metadata: { │ │ "eve.session_id": "", +│ │ "model": "qwen/qwen3-30b-a3b", +│ │ "provider": "openrouter", │ │ "scenario": "eve-instrumentation", │ │ "testRunId": "" │ │ } @@ -95,8 +97,6 @@ span_tree: │ │ output: "Researcher result: {\"title\": \"Eve instrumentation\", \"url\": \"https://eve.dev/docs/guides/instrumentation\"}" │ │ metadata: { │ │ "eve.session_id": "", -│ │ "model": "qwen/qwen3-30b-a3b", -│ │ "provider": "openrouter", │ │ "scenario": "eve-instrumentation", │ │ "testRunId": "" │ │ } @@ -347,6 +347,8 @@ span_tree: │ ] │ metadata: { │ "eve.session_id": "", +│ "model": "qwen/qwen3-30b-a3b", +│ "provider": "openrouter", │ "scenario": "eve-instrumentation", │ "testRunId": "" │ } @@ -498,6 +500,8 @@ span_tree: │ ] │ metadata: { │ "eve.session_id": "", + │ "model": "qwen/qwen3-30b-a3b", + │ "provider": "openrouter", │ "scenario": "eve-instrumentation", │ "testRunId": "" │ } @@ -528,8 +532,6 @@ span_tree: │ output: "Researcher result: Eve instrumentation (https://eve.dev/docs/guides/instrumentation)" │ metadata: { │ "eve.session_id": "", - │ "model": "qwen/qwen3-30b-a3b", - │ "provider": "openrouter", │ "scenario": "eve-instrumentation", │ "testRunId": "" │ } @@ -855,6 +857,8 @@ span_tree: ] metadata: { "eve.session_id": "", + "model": "qwen/qwen3-30b-a3b", + "provider": "openrouter", "scenario": "eve-instrumentation", "testRunId": "" } diff --git a/e2e/scenarios/eve-instrumentation/agent/agent.ts b/e2e/scenarios/eve-instrumentation/agent/agent.ts index 589c856cc..8b518295b 100644 --- a/e2e/scenarios/eve-instrumentation/agent/agent.ts +++ b/e2e/scenarios/eve-instrumentation/agent/agent.ts @@ -18,11 +18,15 @@ const dynamicModel = withReadableReasoning( ); export default defineAgent({ + experimental: { + instrumentationProviders: true, + }, model: defineDynamic({ - fallback: dynamicModel, events: { - "step.started": () => dynamicModel, + "step.started": () => ({ + model: dynamicModel, + modelContextWindowTokens: 8_192, + }), }, }), - modelContextWindowTokens: 8_192, }); diff --git a/e2e/scenarios/eve-instrumentation/agent/channels/eve.ts b/e2e/scenarios/eve-instrumentation/agent/channels/eve.ts new file mode 100644 index 000000000..b74a16488 --- /dev/null +++ b/e2e/scenarios/eve-instrumentation/agent/channels/eve.ts @@ -0,0 +1,7 @@ +import { none } from "eve/channels/auth"; +import { eveChannel } from "eve/channels/eve"; + +export default eveChannel({ + // This local e2e fixture has no user accounts and only serves the test runner. + auth: none(), +}); diff --git a/e2e/scenarios/eve-instrumentation/agent/hooks/braintrust.ts b/e2e/scenarios/eve-instrumentation/agent/hooks/braintrust.ts deleted file mode 100644 index fcac75fde..000000000 --- a/e2e/scenarios/eve-instrumentation/agent/hooks/braintrust.ts +++ /dev/null @@ -1,15 +0,0 @@ -import { braintrustEveHook } from "braintrust"; -import { defineState } from "eve/context"; -import { defineHook } from "eve/hooks"; - -export default defineHook( - braintrustEveHook({ - defineState, - metadata: { - scenario: "eve-instrumentation", - ...(process.env.BRAINTRUST_E2E_RUN_ID - ? { testRunId: process.env.BRAINTRUST_E2E_RUN_ID } - : {}), - }, - }), -); diff --git a/e2e/scenarios/eve-instrumentation/agent/instrumentation.ts b/e2e/scenarios/eve-instrumentation/agent/instrumentation/braintrust.ts similarity index 66% rename from e2e/scenarios/eve-instrumentation/agent/instrumentation.ts rename to e2e/scenarios/eve-instrumentation/agent/instrumentation/braintrust.ts index fff8a737d..9ba5484f2 100644 --- a/e2e/scenarios/eve-instrumentation/agent/instrumentation.ts +++ b/e2e/scenarios/eve-instrumentation/agent/instrumentation/braintrust.ts @@ -1,10 +1,14 @@ import { braintrustEveInstrumentation, initLogger } from "braintrust"; -import { defineState } from "eve/context"; import { defineInstrumentation } from "eve/instrumentation"; export default defineInstrumentation( braintrustEveInstrumentation({ - defineState, + metadata: { + scenario: "eve-instrumentation", + ...(process.env.BRAINTRUST_E2E_RUN_ID + ? { testRunId: process.env.BRAINTRUST_E2E_RUN_ID } + : {}), + }, setup: ({ agentName }) => { initLogger({ projectName: process.env.BRAINTRUST_E2E_PROJECT_NAME || agentName, diff --git a/e2e/scenarios/eve-instrumentation/agent/subagents/researcher/agent.ts b/e2e/scenarios/eve-instrumentation/agent/subagents/researcher/agent.ts index 955d05a88..158110861 100644 --- a/e2e/scenarios/eve-instrumentation/agent/subagents/researcher/agent.ts +++ b/e2e/scenarios/eve-instrumentation/agent/subagents/researcher/agent.ts @@ -11,6 +11,9 @@ const openrouter = createOpenRouter({ export default defineAgent({ description: "Research the Eve instrumentation documentation before the parent reads it.", + experimental: { + instrumentationProviders: true, + }, model: withReadableReasoning( openrouter("qwen/qwen3-30b-a3b", { provider: { diff --git a/e2e/scenarios/eve-instrumentation/agent/subagents/researcher/hooks/braintrust.ts b/e2e/scenarios/eve-instrumentation/agent/subagents/researcher/hooks/braintrust.ts deleted file mode 100644 index fcac75fde..000000000 --- a/e2e/scenarios/eve-instrumentation/agent/subagents/researcher/hooks/braintrust.ts +++ /dev/null @@ -1,15 +0,0 @@ -import { braintrustEveHook } from "braintrust"; -import { defineState } from "eve/context"; -import { defineHook } from "eve/hooks"; - -export default defineHook( - braintrustEveHook({ - defineState, - metadata: { - scenario: "eve-instrumentation", - ...(process.env.BRAINTRUST_E2E_RUN_ID - ? { testRunId: process.env.BRAINTRUST_E2E_RUN_ID } - : {}), - }, - }), -); diff --git a/e2e/scenarios/eve-instrumentation/agent/subagents/researcher/instrumentation/braintrust.ts b/e2e/scenarios/eve-instrumentation/agent/subagents/researcher/instrumentation/braintrust.ts new file mode 100644 index 000000000..9ba5484f2 --- /dev/null +++ b/e2e/scenarios/eve-instrumentation/agent/subagents/researcher/instrumentation/braintrust.ts @@ -0,0 +1,18 @@ +import { braintrustEveInstrumentation, initLogger } from "braintrust"; +import { defineInstrumentation } from "eve/instrumentation"; + +export default defineInstrumentation( + braintrustEveInstrumentation({ + metadata: { + scenario: "eve-instrumentation", + ...(process.env.BRAINTRUST_E2E_RUN_ID + ? { testRunId: process.env.BRAINTRUST_E2E_RUN_ID } + : {}), + }, + setup: ({ agentName }) => { + initLogger({ + projectName: process.env.BRAINTRUST_E2E_PROJECT_NAME || agentName, + }); + }, + }), +); diff --git a/e2e/scenarios/eve-instrumentation/cassette-filter.mjs b/e2e/scenarios/eve-instrumentation/cassette-filter.mjs index f73d1a5ef..4066cef31 100644 --- a/e2e/scenarios/eve-instrumentation/cassette-filter.mjs +++ b/e2e/scenarios/eve-instrumentation/cassette-filter.mjs @@ -1,5 +1,117 @@ // @ts-check -export { filter } from "../ai-sdk-instrumentation/cassette-filter.mjs"; +const EVE_BUILTIN_TOOLS = new Set([ + "agent", + "ask_question", + "bash", + "glob", + "grep", + "load_skill", + "read_file", + "todo", + "web_fetch", + "write_file", +]); + +/** @type {import("@braintrust/seinfeld").FilterSpec} */ +export const filter = [ + "default", + { + ignoreBodyFields: [ + "experimental_generateMessageId", + "messageId", + "messages.*.id", + "messages.*.experimental_messageId", + // Eve changed the runtime-owned subagent output-schema declaration + // without changing this fixture's authored tool contract. + "tools.*.function.parameters.properties.outputSchema", + // JSON Schema's draft marker is metadata, not part of the accepted input. + "tools.*.function.parameters.$schema", + ], + normalizeRequest(request) { + if ( + request.body.kind !== "json" || + request.body.value === null || + typeof request.body.value !== "object" || + Array.isArray(request.body.value) + ) { + return request; + } + const body = request.body.value; + if (!Array.isArray(body.tools)) { + return request; + } + const messages = Array.isArray(body.messages) + ? body.messages.map((message) => message) + : body.messages; + if (Array.isArray(messages)) { + for (let index = 0; index < messages.length; index++) { + const message = messages[index]; + if ( + message === null || + typeof message !== "object" || + !("role" in message) || + message.role !== "tool" + ) { + continue; + } + let end = index + 1; + while ( + end < messages.length && + messages[end] !== null && + typeof messages[end] === "object" && + "role" in messages[end] && + messages[end].role === "tool" + ) { + end++; + } + messages.splice( + index, + end - index, + ...messages.slice(index, end).sort((left, right) => { + const leftName = + left !== null && + typeof left === "object" && + "name" in left && + typeof left.name === "string" + ? left.name + : ""; + const rightName = + right !== null && + typeof right === "object" && + "name" in right && + typeof right.name === "string" + ? right.name + : ""; + return leftName.localeCompare(rightName); + }), + ); + index = end - 1; + } + } + return { + ...request, + body: { + kind: "json", + value: { + ...body, + messages, + tools: body.tools.filter( + (tool) => + tool === null || + typeof tool !== "object" || + !("function" in tool) || + tool.function === null || + typeof tool.function !== "object" || + !("name" in tool.function) || + typeof tool.function.name !== "string" || + !EVE_BUILTIN_TOOLS.has(tool.function.name), + ), + }, + }, + }; + }, + }, +]; /** @type {import("@braintrust/seinfeld").RedactionSpec} */ export const redact = [ diff --git a/e2e/scenarios/eve-instrumentation/package.json b/e2e/scenarios/eve-instrumentation/package.json index 202841885..93b789343 100644 --- a/e2e/scenarios/eve-instrumentation/package.json +++ b/e2e/scenarios/eve-instrumentation/package.json @@ -14,9 +14,9 @@ }, "dependencies": { "@openrouter/ai-sdk-provider": "3.0.0", - "ai": "7.0.34", - "eve-v0": "npm:eve@0.22.1", - "eve-v0-latest": "npm:eve@0.27.6", + "ai": "7.0.68", + "eve-v0": "npm:eve@0.34.0", + "eve-v0-latest": "npm:eve@0.39.1", "zod": "4.3.6" } } diff --git a/e2e/scenarios/eve-instrumentation/pnpm-lock.yaml b/e2e/scenarios/eve-instrumentation/pnpm-lock.yaml index c3cbc645d..e6b942a7a 100644 --- a/e2e/scenarios/eve-instrumentation/pnpm-lock.yaml +++ b/e2e/scenarios/eve-instrumentation/pnpm-lock.yaml @@ -10,36 +10,36 @@ importers: dependencies: '@openrouter/ai-sdk-provider': specifier: 3.0.0 - version: 3.0.0(ai@7.0.34(zod@4.3.6))(zod@4.3.6) + version: 3.0.0(ai@7.0.68(zod@4.3.6))(zod@4.3.6) ai: - specifier: 7.0.34 - version: 7.0.34(zod@4.3.6) + specifier: 7.0.68 + version: 7.0.68(zod@4.3.6) eve-v0: - specifier: npm:eve@0.22.1 - version: eve@0.22.1(ai@7.0.34(zod@4.3.6)) + specifier: npm:eve@0.34.0 + version: eve@0.34.0(ai@7.0.68(zod@4.3.6)) eve-v0-latest: - specifier: npm:eve@0.27.6 - version: eve@0.27.6(ai@7.0.34(zod@4.3.6)) + specifier: npm:eve@0.39.1 + version: eve@0.39.1(ai@7.0.68(zod@4.3.6)) zod: specifier: 4.3.6 version: 4.3.6 packages: - '@ai-sdk/gateway@4.0.26': - resolution: {integrity: sha512-b/nc3COKtk8IxzgcCi418IoZFky/Bw+Jg8+J0/SBBnu/mOXA4bkIKCu81coEtJAWuH2g5Fvvsa0ar7EpmzNWTw==} + '@ai-sdk/gateway@4.0.54': + resolution: {integrity: sha512-x4fAXDqCtYzB/M5vsIQLYcyrzpJuaRgcIwDSw+lpTMMbgH19fU3ds75GSlHNLzfx6Z5yL4Z9+EMr0GJcqVy9QA==} engines: {node: '>=22'} peerDependencies: zod: ^3.25.76 || ^4.1.8 - '@ai-sdk/provider-utils@5.0.12': - resolution: {integrity: sha512-bbhlOgHeYwrIGheLkM6fhS8hVger8uFPmcOLg+kxc9EFh7y30XYorWhthlYAgpadO3SJhFZrIcEknN7qEqEVvA==} + '@ai-sdk/provider-utils@5.0.27': + resolution: {integrity: sha512-EzAn4pdgG5g0xXtH6lE2zyNmfjDQIDjATkfqzuidEI35g++hh4+07vnjzkT/RmGmIClPZiRj/Q2GMPV2V7mkHw==} engines: {node: '>=22'} peerDependencies: zod: ^3.25.76 || ^4.1.8 - '@ai-sdk/provider@4.0.3': - resolution: {integrity: sha512-e0CpNWJUY7OxAFAnCZkw+ri9QOHWwTs1tXP42782KFGCU07qt8NiXCrCVowyCB5dP2r5/Uls+g2oPd8kOJn9dw==} + '@ai-sdk/provider@4.0.7': + resolution: {integrity: sha512-6or44XprPzKbr8zkmzosowSE0pxkvJcoojBL+mCZvPUt3kvXp3XSNqeVun9golb1acEfSo6yaEBRT18h2VU+1Q==} engines: {node: '>=22'} '@emnapi/core@1.11.1': @@ -178,8 +178,8 @@ packages: '@workflow/serde@4.1.0': resolution: {integrity: sha512-pav4F2BoirECWR7Nf1TKt+2eETcBj7jj4cBefQ8VXQCA6NPkaKeLfj/zMgi+3zYV5ZIBT4GuUiphsj0/b9hPQQ==} - ai@7.0.34: - resolution: {integrity: sha512-jDqclWYqPGFKcUG4CQiKcBiwi5XqVMqvYy6eJdujVpvE1SDoB6j7RtjrGYz3Bn5B1dzTj1StAlrOebY//WK9/Q==} + ai@7.0.68: + resolution: {integrity: sha512-9QuZOT77wzoxxUC0NcueXhCo3HUHA/1pApIJ9VRyE+9/K+3Innkq4kVhrd9aEnwIviJz2Nga063m+UTsPSdOyw==} engines: {node: '>=22'} peerDependencies: zod: ^3.25.76 || ^4.1.8 @@ -237,13 +237,13 @@ packages: wrangler: optional: true - eve@0.22.1: - resolution: {integrity: sha512-4nqy8SEJpS94fM8sxd0Or+irvf/CmS0qENxouKyyegst3Qtn06xZbuSEoI1p7i4wB6sprr95udXh7Ulct6ABrA==} + eve@0.34.0: + resolution: {integrity: sha512-zgrYIbm8u78Furxfdyzaknf5AxFCzwmTGfLEvgax00BUNXu3eebaos8igjYQyM4q5JAxlWAoZCJuItE6r2S0cw==} engines: {node: '>=24'} hasBin: true peerDependencies: '@opentelemetry/api': ^1.0.0 - ai: ^7.0.0 + ai: ^7.0.58 braintrust: ^3.0.0 just-bash: ^3.0.0 microsandbox: ^0.5.0 @@ -257,13 +257,13 @@ packages: microsandbox: optional: true - eve@0.27.6: - resolution: {integrity: sha512-t2Xr/nlVGc0+Fy9fav8p/CJrwMSQM0zyuOGY1F1XapN0UFssGYBGa6Ml/Q5OnowvWwsGH4m0Tf2QAccayVCvtw==} + eve@0.39.1: + resolution: {integrity: sha512-HTd1A3/GsRKNpJ+6WEW4OEPAhWAId5IZ769hjuY3eJqj3cWeaEvBUDZSjUFjSDit63aqbfur0u7AH1m0JNr7EQ==} engines: {node: '>=24'} hasBin: true peerDependencies: '@opentelemetry/api': ^1.0.0 - ai: ^7.0.34 + ai: ^7.0.58 braintrust: ^3.0.0 just-bash: ^3.0.0 microsandbox: ^0.5.0 @@ -365,6 +365,14 @@ packages: tslib@2.8.1: resolution: {integrity: sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==} + undici@7.29.0: + resolution: {integrity: sha512-IDxfleLmmbSskfWSUATiN1nfn2rDuvnMOqb5CWR92iIfojA0Ud+ulOAAEQ57LPr9rWmsreUyf5lwyao+7GNNVw==} + engines: {node: '>=20.18.1'} + + undici@8.9.0: + resolution: {integrity: sha512-aWZpUj7XoGonMClx4gdDRfgBjqeA+F473aDmROQQbM9n6PRfK/u1q/a0X4wMTgcHfT8H6fpbt98PFuDUwFg2YA==} + engines: {node: '>=22.19.0'} + unenv@2.0.0-rc.24: resolution: {integrity: sha512-i7qRCmY42zmCwnYlh9H2SvLEypEFGye5iRmEMKjcGi7zk9UquigRjFtTLz0TYqr0ZGLZhaMHl/foy1bZR+Cwlw==} @@ -447,22 +455,23 @@ packages: snapshots: - '@ai-sdk/gateway@4.0.26(zod@4.3.6)': + '@ai-sdk/gateway@4.0.54(zod@4.3.6)': dependencies: - '@ai-sdk/provider': 4.0.3 - '@ai-sdk/provider-utils': 5.0.12(zod@4.3.6) + '@ai-sdk/provider': 4.0.7 + '@ai-sdk/provider-utils': 5.0.27(zod@4.3.6) '@vercel/oidc': 3.2.0 zod: 4.3.6 - '@ai-sdk/provider-utils@5.0.12(zod@4.3.6)': + '@ai-sdk/provider-utils@5.0.27(zod@4.3.6)': dependencies: - '@ai-sdk/provider': 4.0.3 + '@ai-sdk/provider': 4.0.7 '@standard-schema/spec': 1.1.0 '@workflow/serde': 4.1.0 eventsource-parser: 3.1.0 + undici: 7.29.0 zod: 4.3.6 - '@ai-sdk/provider@4.0.3': + '@ai-sdk/provider@4.0.7': dependencies: json-schema: 0.4.0 @@ -489,9 +498,9 @@ snapshots: '@tybys/wasm-util': 0.10.3 optional: true - '@openrouter/ai-sdk-provider@3.0.0(ai@7.0.34(zod@4.3.6))(zod@4.3.6)': + '@openrouter/ai-sdk-provider@3.0.0(ai@7.0.68(zod@4.3.6))(zod@4.3.6)': dependencies: - ai: 7.0.34(zod@4.3.6) + ai: 7.0.68(zod@4.3.6) zod: 4.3.6 '@oxc-project/types@0.138.0': {} @@ -558,11 +567,11 @@ snapshots: '@workflow/serde@4.1.0': {} - ai@7.0.34(zod@4.3.6): + ai@7.0.68(zod@4.3.6): dependencies: - '@ai-sdk/gateway': 4.0.26(zod@4.3.6) - '@ai-sdk/provider': 4.0.3 - '@ai-sdk/provider-utils': 5.0.12(zod@4.3.6) + '@ai-sdk/gateway': 4.0.54(zod@4.3.6) + '@ai-sdk/provider': 4.0.7 + '@ai-sdk/provider-utils': 5.0.27(zod@4.3.6) zod: 4.3.6 consola@3.4.2: {} @@ -580,10 +589,11 @@ snapshots: httpxy: 0.5.4 srvx: 0.11.21 - eve@0.22.1(ai@7.0.34(zod@4.3.6)): + eve@0.34.0(ai@7.0.68(zod@4.3.6)): dependencies: - ai: 7.0.34(zod@4.3.6) + ai: 7.0.68(zod@4.3.6) nitro: 3.0.260610-beta + undici: 8.9.0 transitivePeerDependencies: - '@azure/app-configuration' - '@azure/cosmos' @@ -624,10 +634,11 @@ snapshots: - xml2js - zephyr-agent - eve@0.27.6(ai@7.0.34(zod@4.3.6)): + eve@0.39.1(ai@7.0.68(zod@4.3.6)): dependencies: - ai: 7.0.34(zod@4.3.6) + ai: 7.0.68(zod@4.3.6) nitro: 3.0.260610-beta + undici: 8.9.0 transitivePeerDependencies: - '@azure/app-configuration' - '@azure/cosmos' @@ -773,6 +784,10 @@ snapshots: tslib@2.8.1: optional: true + undici@7.29.0: {} + + undici@8.9.0: {} + unenv@2.0.0-rc.24: dependencies: pathe: 2.0.3 diff --git a/e2e/scenarios/eve-instrumentation/scenario.test.ts b/e2e/scenarios/eve-instrumentation/scenario.test.ts index 4d65888c9..2fcd1ad48 100644 --- a/e2e/scenarios/eve-instrumentation/scenario.test.ts +++ b/e2e/scenarios/eve-instrumentation/scenario.test.ts @@ -26,7 +26,7 @@ const eveScenarios = await Promise.all( [ { dependencyName: "eve-v0", - label: "v0 pinned", + label: "v0.34 pinned", variantKey: "eve-v0", }, { @@ -137,12 +137,16 @@ describe.sequential("eve instrumentation variants", () => { "read", secondRoot?.span.id, ); - + const secondChildSearch = findLatestChildSpan( + events, + "search", + secondChildTurn?.span.id, + ); expect(findAllSpans(events, "eve.session")).toEqual([]); expect(turns).toHaveLength(4); expect( turns.filter((turn) => turn.span.parentIds.length === 0), - ).toHaveLength(2); + ).toEqual([root, secondRoot]); expect(new Set(turns.map((turn) => turn.span.rootId)).size).toBe(2); expect(root).toBeDefined(); @@ -155,10 +159,21 @@ describe.sequential("eve instrumentation variants", () => { }); expect(root?.metadata).not.toHaveProperty("model"); expect(root?.metadata).not.toHaveProperty("provider"); - expect(root?.metrics?.completion_tokens).toEqual(expect.any(Number)); - expect(root?.metrics?.prompt_tokens).toEqual(expect.any(Number)); - expect(root?.metrics?.tokens).toEqual(expect.any(Number)); expect(root?.output).toContain("Final answer from read"); + expect(root?.metrics).toMatchObject({ + completion_tokens: steps.reduce( + (total, step) => total + (step.metrics?.completion_tokens ?? 0), + 0, + ), + prompt_tokens: steps.reduce( + (total, step) => total + (step.metrics?.prompt_tokens ?? 0), + 0, + ), + tokens: steps.reduce( + (total, step) => total + (step.metrics?.tokens ?? 0), + 0, + ), + }); expect(steps).toHaveLength(2); expect(steps.map((step) => step.span.type)).toEqual(["llm", "llm"]); @@ -191,11 +206,14 @@ describe.sequential("eve instrumentation variants", () => { expect(step.input[0]).toMatchObject({ role: "system" }); expect(step.metadata).toMatchObject({ "eve.session_id": root?.metadata?.["eve.session_id"], + model: "qwen/qwen3-30b-a3b", + provider: "openrouter", scenario: "eve-instrumentation", testRunId: expect.any(String), }); - expect(step.metadata).not.toHaveProperty("model"); - expect(step.metadata).not.toHaveProperty("provider"); + expect(step.metrics?.completion_tokens).toEqual(expect.any(Number)); + expect(step.metrics?.prompt_tokens).toEqual(expect.any(Number)); + expect(step.metrics?.tokens).toEqual(expect.any(Number)); } expect(researcher).toBeDefined(); @@ -217,14 +235,28 @@ describe.sequential("eve instrumentation variants", () => { expect(childTurn?.span.rootId).toEqual(root?.span.rootId); expect(childTurn?.metadata).toMatchObject({ "eve.session_id": expect.any(String), - model: "qwen/qwen3-30b-a3b", - provider: "openrouter", scenario: "eve-instrumentation", testRunId: expect.any(String), }); + expect(childTurn?.metadata).not.toHaveProperty("model"); + expect(childTurn?.metadata).not.toHaveProperty("provider"); expect(childTurn?.metadata?.["eve.session_id"]).not.toEqual( root?.metadata?.["eve.session_id"], ); + expect(childTurn?.metrics).toMatchObject({ + completion_tokens: childSteps.reduce( + (total, step) => total + (step.metrics?.completion_tokens ?? 0), + 0, + ), + prompt_tokens: childSteps.reduce( + (total, step) => total + (step.metrics?.prompt_tokens ?? 0), + 0, + ), + tokens: childSteps.reduce( + (total, step) => total + (step.metrics?.tokens ?? 0), + 0, + ), + }); expect(childSteps).toHaveLength(2); for (const step of childSteps) { @@ -242,6 +274,9 @@ describe.sequential("eve instrumentation variants", () => { scenario: "eve-instrumentation", testRunId: expect.any(String), }); + expect(step.metrics?.completion_tokens).toEqual(expect.any(Number)); + expect(step.metrics?.prompt_tokens).toEqual(expect.any(Number)); + expect(step.metrics?.tokens).toEqual(expect.any(Number)); expect(step.output).toMatchObject([ { message: { @@ -286,74 +321,81 @@ describe.sequential("eve instrumentation variants", () => { expect(secondRoot?.span.rootId).not.toEqual(root?.span.rootId); expect(secondRoot?.metadata).toMatchObject({ "eve.session_id": root?.metadata?.["eve.session_id"], + scenario: "eve-instrumentation", + testRunId: expect.any(String), }); - expect(secondRoot?.metadata).not.toHaveProperty("model"); - expect(secondRoot?.metadata).not.toHaveProperty("provider"); expect(secondRoot?.output).toContain("Final answer from read"); + expect(secondRoot?.metrics).toMatchObject({ + completion_tokens: secondSteps.reduce( + (total, step) => total + (step.metrics?.completion_tokens ?? 0), + 0, + ), + prompt_tokens: secondSteps.reduce( + (total, step) => total + (step.metrics?.prompt_tokens ?? 0), + 0, + ), + tokens: secondSteps.reduce( + (total, step) => total + (step.metrics?.tokens ?? 0), + 0, + ), + }); expect(secondSteps).toHaveLength(2); - expect(secondSteps.map((step) => step.span.type)).toEqual([ - "llm", - "llm", - ]); for (const step of secondSteps) { + expect(step.span.parentIds).toEqual([secondRoot?.span.id]); + expect(step.span.rootId).toEqual(secondRoot?.span.rootId); expect(step.metadata).toMatchObject({ "eve.session_id": secondRoot?.metadata?.["eve.session_id"], - scenario: "eve-instrumentation", - testRunId: expect.any(String), + model: "qwen/qwen3-30b-a3b", + provider: "openrouter", }); - expect(step.metadata).not.toHaveProperty("model"); - expect(step.metadata).not.toHaveProperty("provider"); } - expect(secondSteps[0]?.output).toMatchObject([ - { - finish_reason: "tool_calls", - message: { - reasoning: [{ content: expect.any(String) }], - tool_calls: [ - { function: { name: "researcher" }, type: "function" }, - { function: { name: "read" }, type: "function" }, - ], - }, - }, - ]); - expect(secondSteps[1]?.output).toMatchObject([ - { - finish_reason: "stop", - message: { - reasoning: [{ content: expect.any(String) }], - }, - }, - ]); - expect(secondResearcher?.span.type).toBe("tool"); - expect(secondResearcher?.span.ended).toBe(true); expect(secondResearcher?.span.parentIds).toEqual([secondRoot?.span.id]); - expect(secondResearcher?.metadata).toMatchObject({ - "eve.session_id": secondRoot?.metadata?.["eve.session_id"], - }); expect(secondChildTurn?.span.parentIds).toEqual([ secondResearcher?.span.id, ]); expect(secondChildTurn?.span.rootId).toEqual(secondRoot?.span.rootId); - expect(secondChildTurn?.metadata).toMatchObject({ - "eve.session_id": expect.any(String), - model: "qwen/qwen3-30b-a3b", - provider: "openrouter", - }); - expect(secondChildTurn?.metadata?.["eve.session_id"]).not.toEqual( - secondRoot?.metadata?.["eve.session_id"], - ); - expect(secondRead?.span.type).toBe("tool"); - expect(secondRead?.span.ended).toBe(true); + expect(secondChildTurn?.span.rootId).not.toEqual(root?.span.rootId); expect(secondRead?.span.parentIds).toEqual([secondRoot?.span.id]); - expect(secondRead?.metadata).toMatchObject({ - "eve.session_id": secondRoot?.metadata?.["eve.session_id"], - }); for (const event of events) { expect(spanInstrumentationName(event)).toBe("eve"); } const rawRows = payloads.flatMap((payload) => payload.rows); + for (const span of [ + ...findAllSpans(events, "eve.step"), + researcher, + childSearch, + read, + secondResearcher, + secondRead, + secondChildSearch, + ]) { + expect(span).toBeDefined(); + expect( + rawRows.filter( + (row) => + row.id === span?.row.id && + Object.prototype.hasOwnProperty.call(row, "input"), + ), + ).toHaveLength(1); + expect( + rawRows.filter( + (row) => + row.id === span?.row.id && + Object.prototype.hasOwnProperty.call(row, "metadata"), + ), + ).toHaveLength(1); + } + for (const turn of turns) { + expect( + rawRows.filter( + (row) => + row.id === turn.row.id && + Object.prototype.hasOwnProperty.call(row, "metadata"), + ), + ).toHaveLength(1); + } for (const step of findAllSpans(events, "eve.step")) { expect( rawRows.filter( diff --git a/e2e/scenarios/eve-instrumentation/scenario.ts b/e2e/scenarios/eve-instrumentation/scenario.ts index 48efc7bea..ca1c92027 100644 --- a/e2e/scenarios/eve-instrumentation/scenario.ts +++ b/e2e/scenarios/eve-instrumentation/scenario.ts @@ -65,13 +65,10 @@ async function main() { } const body = (await response.json()) as { - continuationToken?: string; sessionId?: string; }; - if (!body.sessionId || !body.continuationToken) { - throw new Error( - `Eve session create did not return a sessionId and continuationToken`, - ); + if (!body.sessionId) { + throw new Error(`Eve session create did not return a sessionId`); } const seenSessionIds = new Set([body.sessionId]); @@ -81,11 +78,13 @@ async function main() { seenSessionIds, "session.waiting", ); + // Eve emits session.waiting just before its durable session snapshot is + // visible to the continuation route. + await new Promise((resolve) => setTimeout(resolve, 1000)); const followUp = await fetch( `${baseUrl}/eve/v1/session/${body.sessionId}`, { body: JSON.stringify({ - continuationToken: body.continuationToken, message: "Run the Braintrust Eve instrumentation e2e scenario again", }), headers: { "content-type": "application/json" }, @@ -213,6 +212,7 @@ async function streamUntil( const decoder = new TextDecoder(); let buffer = ""; let nextIndex = startIndex; + let turnCompleted = false; try { while (true) { const { done, value } = await reader.read(); @@ -228,7 +228,10 @@ async function streamUntil( continue; } const event = JSON.parse(trimmed) as { - data?: { childSessionId?: string; message?: string }; + data?: { + childSessionId?: string; + message?: string; + }; type?: string; }; nextIndex++; @@ -256,7 +259,13 @@ async function streamUntil( ).then(() => undefined), ); } - if (event.type === until) { + if (event.type === "turn.completed") { + turnCompleted = true; + } + if ( + event.type === until && + (until !== "session.waiting" || turnCompleted) + ) { await Promise.all(childStreams); return nextIndex; } diff --git a/js/src/exports.ts b/js/src/exports.ts index d39c15459..9ed496ceb 100644 --- a/js/src/exports.ts +++ b/js/src/exports.ts @@ -196,10 +196,7 @@ export { BraintrustMiddleware, wrapAISDKModel, } from "./wrappers/ai-sdk"; -export { - braintrustEveHook, - braintrustEveInstrumentation, -} from "./instrumentation/plugins/eve-plugin"; +export { braintrustEveInstrumentation } from "./instrumentation/plugins/eve-plugin"; export { collectAnthropicSession } from "./wrappers/anthropic-session-collector"; export { wrapAnthropic } from "./wrappers/anthropic"; export { diff --git a/js/src/instrumentation/index.ts b/js/src/instrumentation/index.ts index ce4352ebc..8bbbd31ec 100644 --- a/js/src/instrumentation/index.ts +++ b/js/src/instrumentation/index.ts @@ -20,10 +20,7 @@ export type { BraintrustPluginConfig } from "./braintrust-plugin"; export { OpenAIAgentsTraceProcessor } from "./plugins/openai-agents-trace-processor"; export type { OpenAIAgentsTraceProcessorOptions } from "./plugins/openai-agents-trace-processor"; export { braintrustFlueInstrumentation } from "./plugins/flue-plugin"; -export { - braintrustEveHook, - braintrustEveInstrumentation, -} from "./plugins/eve-plugin"; +export { braintrustEveInstrumentation } from "./plugins/eve-plugin"; // Re-export core types for external instrumentation packages export type { diff --git a/js/src/instrumentation/plugins/eve-plugin.test.ts b/js/src/instrumentation/plugins/eve-plugin.test.ts index bf4323b71..38e15bef0 100644 --- a/js/src/instrumentation/plugins/eve-plugin.test.ts +++ b/js/src/instrumentation/plugins/eve-plugin.test.ts @@ -7,46 +7,38 @@ import { it, vi, } from "vitest"; -import { createHash } from "node:crypto"; import * as braintrustExports from "../../exports"; import { configureNode } from "../../node/config"; -import { - _exportsForTestingOnly, - initLogger, - startSpan, - withCurrent, -} from "../../logger"; +import { Attachment, _exportsForTestingOnly, initLogger } from "../../logger"; import * as instrumentationExports from "../index"; -import { braintrustEveHook, braintrustEveInstrumentation } from "./eve-plugin"; +import { braintrustEveInstrumentation } from "./eve-plugin"; import type { - EveHandleMessageStreamEvent, - EveHookContext, + EveInstrumentationAttemptScope, + EveInstrumentationHandlerContext, + EveInstrumentationModelCallCompletedEvent, + EveJsonValue, } from "../../vendor-sdk-types/eve"; -import { mergeRowBatch } from "../../../util/index"; - -function deterministicEveIdForTest(...parts: string[]): string { - return createHash("sha256") - .update(parts.map((part) => `${part.length}:${part}`).join("\0")) - .digest("hex") - .slice(0, 32) - .replace(/^(.{8})(.{4})(.{4})(.{4})(.{12})$/, "$1-$2-$3-$4-$5"); -} -function createFakeDefineState() { - const values = new Map(); +const ATTEMPT_SCOPE: EveInstrumentationAttemptScope = { + attemptId: "attempt-0", + attemptIndex: 0, + rootSessionId: "session-root", + sessionId: "session-root", + stepIndex: 0, + turnId: "turn-0", +}; + +function providerContext( + initial?: EveJsonValue, +): EveInstrumentationHandlerContext { + let value = initial; return { - defineState(name: string, initial: () => T) { - return { - get: () => (values.has(name) ? (values.get(name) as T) : initial()), - update: (fn: (current: T) => T) => { - values.set( - name, - fn(values.has(name) ? (values.get(name) as T) : initial()), - ); - }, - }; + state: { + get: () => value, + set: (next) => { + value = next; + }, }, - values, }; } @@ -56,18 +48,16 @@ try { // Best-effort initialization for test environments. } -describe("braintrustEveHook", () => { +describe("braintrustEveInstrumentation", () => { let backgroundLogger: ReturnType< typeof _exportsForTestingOnly.useTestBackgroundLogger >; - let defineState: ReturnType["defineState"]; beforeAll(async () => { await _exportsForTestingOnly.simulateLoginForTests(); }); beforeEach(() => { - defineState = createFakeDefineState().defineState; backgroundLogger = _exportsForTestingOnly.useTestBackgroundLogger(); initLogger({ projectName: "eve-plugin.test.ts", @@ -80,517 +70,186 @@ describe("braintrustEveHook", () => { _exportsForTestingOnly.clearTestBackgroundLogger(); }); - it("returns an Eve hook definition", () => { - const hook = braintrustEveHook({ defineState }); - - expect(Object.keys(hook)).toEqual(["events"]); - expect(typeof hook.events?.["*"]).toBe("function"); - }); - - it("returns an Eve instrumentation definition", () => { + it("returns an Eve 0.34+ instrumentation provider", () => { const setup = vi.fn(); - const instrumentation = braintrustEveInstrumentation({ - defineState, - setup, - }); - - expect(instrumentation).toMatchObject({ - recordInputs: false, - recordOutputs: false, - setup, - }); - expect(typeof instrumentation.events?.["step.started"]).toBe("function"); - }); - - it("requires Eve's defineState API", () => { - expect(() => braintrustEveHook(undefined as never)).toThrow(); - expect(() => braintrustEveInstrumentation(undefined as never)).toThrow(); + const provider = braintrustEveInstrumentation({ setup }); + + expect(provider.capture).toBe("content"); + expect(provider.setup).toBe(setup); + expect(typeof provider.flush).toBe("function"); + expect(Object.keys(provider.events).sort()).toEqual([ + "action.completed", + "action.failed", + "action.started", + "model.call.completed", + "model.call.failed", + "model.call.started", + "step.attempt.completed", + "step.attempt.failed", + "turn.cancelled", + "turn.completed", + "turn.failed", + "turn.started", + ]); }); - it("exports Eve APIs from root and instrumentation entrypoints", () => { - expect(braintrustExports.braintrustEveHook).toBe(braintrustEveHook); + it("exports only the Eve instrumentation provider API", () => { expect(braintrustExports.braintrustEveInstrumentation).toBe( braintrustEveInstrumentation, ); - expect(instrumentationExports.braintrustEveHook).toBe(braintrustEveHook); expect(instrumentationExports.braintrustEveInstrumentation).toBe( braintrustEveInstrumentation, ); + expect(braintrustExports).not.toHaveProperty("braintrustEveHook"); + expect(instrumentationExports).not.toHaveProperty("braintrustEveHook"); }); - it("captures stable model input without provider options", async () => { - const fakeEve = createFakeDefineState(); - defineState = fakeEve.defineState; - const instrumentation = braintrustEveInstrumentation({ defineState }); - const wildcard = braintrustEveHook({ defineState }).events?.["*"]; - expect(wildcard).toBeDefined(); - - const ctx: EveHookContext = { - session: { id: "session-captured-input" }, - }; - const emit = (event: EveHandleMessageStreamEvent) => wildcard?.(event, ctx); - const modelInput = { - instructions: [ - { - content: "Answer with the relevant Eve instrumentation detail.", - providerOptions: { - openai: { reasoningEncryptedContent: "system-provider-secret" }, - }, - role: "system", - }, - { - content: "Keep the answer concise.", - role: "system", - }, - ], - messages: [ - { - content: [ - { - providerOptions: { - openai: { - itemId: "reasoning-item", - reasoningEncryptedContent: "must-not-be-logged", - }, - }, - text: "Readable reasoning", - type: "reasoning", - }, - { - input: { - providerOptions: { application: "tool-input" }, - reasoningEncryptedContent: "legitimate-tool-input", - }, - providerOptions: { - openai: { reasoningEncryptedContent: "tool-provider-secret" }, - }, - toolCallId: "call-1", - toolName: "lookup", - type: "tool-call", - }, - { - output: { - providerOptions: { - openai: { - reasoningEncryptedContent: "output-provider-secret", - }, - }, - type: "json", - value: { - reasoningEncryptedContent: "legitimate-tool-output", - }, - }, - providerOptions: { - openai: { - reasoningEncryptedContent: "result-provider-secret", - }, - }, - toolCallId: "call-1", - toolName: "lookup", - type: "tool-result", - }, - ], - providerOptions: { - openai: { reasoningEncryptedContent: "message-provider-secret" }, - }, - role: "assistant", - }, - ], - } as const; - - await emit({ - data: { - runtime: { - agentId: "agent-captured-input", - eveVersion: "0.22.1", - modelId: "dynamic:anthropic/claude-sonnet-5", - }, - }, - type: "session.started", - }); - await emit({ - data: { sequence: 0, turnId: "turn-captured-input" }, - type: "turn.started", - }); - instrumentation.events?.["step.started"]?.({ - modelInput, - session: { id: "session-captured-input" }, - step: { index: 0 }, - turn: { id: "turn-captured-input", sequence: 0 }, - }); - await emit({ - data: { sequence: 0, stepIndex: 0, turnId: "turn-captured-input" }, - type: "step.started", - }); - await emit({ - data: { - finishReason: "stop", - message: "Capture the model input.", - sequence: 0, - stepIndex: 0, - turnId: "turn-captured-input", - }, - type: "message.completed", - }); - await emit({ - data: { - finishReason: "stop", - sequence: 0, - stepIndex: 0, - turnId: "turn-captured-input", + it("records a turn with LLM and action spans", async () => { + const provider = braintrustEveInstrumentation({ + metadata: { + scenario: "eve-plugin-unit", + testRunId: "test-run-flat-tree", }, - type: "step.completed", - }); - await emit({ - data: { sequence: 0, turnId: "turn-captured-input" }, - type: "turn.completed", }); + const turnContext = providerContext(); + const firstModelContext = providerContext(); + const actionContext = providerContext(); + const secondModelContext = providerContext(); + const scope = ATTEMPT_SCOPE; - const spans = (await backgroundLogger.drain()) as Array< - Record - >; - const step = spans.find( - (span) => span.span_attributes?.name === "eve.step", - ); - const session = spans.find( - (span) => span.span_attributes?.name === "eve.session", - ); - const turn = spans.find( - (span) => span.span_attributes?.name === "eve.turn", - ); - for (const span of [turn, step]) { - expect(span?.context?.span_origin).toMatchObject({ - instrumentation: { name: "eve" }, - }); - } - expect(step?.input).toEqual([ + await provider.events["turn.started"]( { - content: "Answer with the relevant Eve instrumentation detail.", - role: "system", + idempotencyKey: "turn:session-root:turn-0", + rootSessionId: "session-root", + sequence: 0, + sessionId: "session-root", + turnId: "turn-0", + type: "turn.started", }, + turnContext, + ); + await provider.events["model.call.started"]( { - content: "Keep the answer concise.", - role: "system", + idempotencyKey: "model:attempt-0:0", + input: { + instructions: "You are a research agent.", + messages: [{ content: "Search then read", role: "user" }], + }, + model: { + modelId: "qwen/qwen3-30b-a3b", + provider: "openrouter.chat", + }, + scope, + type: "model.call.started", }, + firstModelContext, + ); + await provider.events["model.call.completed"]( { content: [ + { text: "I should search first.", type: "reasoning" }, { - text: "Readable reasoning", - type: "reasoning", - }, - { - input: { - providerOptions: { application: "tool-input" }, - reasoningEncryptedContent: "legitimate-tool-input", - }, - toolCallId: "call-1", - toolName: "lookup", + callId: "call-search", + input: { query: "Eve instrumentation" }, + toolName: "search", type: "tool-call", }, - { - output: { - type: "json", - value: { - reasoningEncryptedContent: "legitimate-tool-output", - }, - }, - toolCallId: "call-1", - toolName: "lookup", - type: "tool-result", - }, ], - role: "assistant", - }, - ]); - expect(JSON.stringify(step?.input)).not.toContain("provider-secret"); - expect(JSON.stringify(step?.input)).not.toContain("must-not-be-logged"); - expect(step?.metadata).toEqual({ - "eve.session_id": "session-captured-input", - }); - expect(session).toBeUndefined(); - expect(turn?.metadata).toEqual({ - "eve.session_id": "session-captured-input", - }); - expect(step?.metadata).not.toHaveProperty("model"); - expect(step?.metadata).not.toHaveProperty("provider"); - expect(turn?.metadata).not.toHaveProperty("model"); - expect(turn?.metadata).not.toHaveProperty("provider"); - expect(fakeEve.values.get("braintrust.eve.tracing")).toMatchObject({ - llmInputs: [], - }); - }); - - it("skips missing or malformed Eve instrumentation state without throwing", async () => { - const instrumentation = braintrustEveInstrumentation({ defineState }); - expect(() => - instrumentation.events?.["step.started"]?.({ bad: true } as never), - ).not.toThrow(); - - const fakeEve = createFakeDefineState(); - defineState = fakeEve.defineState; - fakeEve.values.set("braintrust.eve.tracing", { - llmInputs: [{ input: { content: "not an array" }, key: "bad" }], - }); - expect(() => - instrumentation.events?.["step.started"]?.({ - modelInput: { - messages: [{ content: "hello", role: "user" }], + finishReason: "tool-calls", + idempotencyKey: "model:attempt-0:0", + scope, + type: "model.call.completed", + usage: { + inputTokenDetails: { + cacheReadTokens: 3, + cacheWriteTokens: 2, + }, + inputTokens: 10, + outputTokens: 5, }, - session: { id: "session-malformed-state" }, - step: { index: 0 }, - turn: { id: "turn-malformed-state", sequence: 0 }, - }), - ).not.toThrow(); - - const wildcard = braintrustEveHook({ defineState }).events?.["*"]; - const ctx: EveHookContext = { session: { id: "session-no-state" } }; - await wildcard?.( - { - data: { sequence: 0, turnId: "turn-no-state" }, - type: "turn.started", }, - ctx, + firstModelContext, ); - await wildcard?.( + await provider.events["action.started"]( { - data: { sequence: 0, stepIndex: 0, turnId: "turn-no-state" }, - type: "step.started", + callId: "call-search", + idempotencyKey: "action:session-root:turn-0:call-search", + input: { query: "Eve instrumentation" }, + name: "search", + scope, + type: "action.started", }, - ctx, + actionContext, ); - await wildcard?.( + await provider.events["action.completed"]( { - data: { - finishReason: "stop", - sequence: 0, - stepIndex: 0, - turnId: "turn-no-state", + idempotencyKey: "action:session-root:turn-0:call-search", + output: { + output: { hits: ["eve.dev/docs"] }, + type: "result", }, - type: "step.completed", - }, - ctx, - ); - await wildcard?.( - { - data: { sequence: 0, turnId: "turn-no-state" }, - type: "turn.completed", + scope, + type: "action.completed", }, - ctx, - ); - - const spans = (await backgroundLogger.drain()) as Array< - Record - >; - const step = spans.find( - (span) => span.span_attributes?.name === "eve.step", + actionContext, ); - expect(step?.input).toBeUndefined(); - }); - - it("bounds pre-existing durable trace state", async () => { - const fakeEve = createFakeDefineState(); - const oversizedEntryCount = 10_001; - fakeEve.values.set("braintrust.eve.tracing", { - llmInputs: [], - metadata: {}, - spanReferences: Array.from( - { length: oversizedEntryCount }, - (_, index) => ({ - exported: `exported-${index}`, - rootSpanId: `root-${index}`, - rowId: `row-${index}`, - spanId: `span-${index}`, - }), - ), - stepStarts: Array.from({ length: oversizedEntryCount }, (_, index) => ({ - open: false, - ordinal: index, - stepIndex: index, - turnId: `turn-${index}`, - })), - }); - const wildcard = braintrustEveHook({ - defineState: fakeEve.defineState, - }).events?.["*"]; - - await wildcard?.( + await provider.events["model.call.started"]( { - data: { - runtime: { - agentId: "agent-bounded-state", - eveVersion: "0.20.0", - modelId: "openai/gpt-5.4-mini", - }, + idempotencyKey: "model:attempt-1:0", + input: { + instructions: "You are a research agent.", + messages: [ + { content: "Search then read", role: "user" }, + { + content: [ + { + input: { query: "Eve instrumentation" }, + toolCallId: "call-search", + toolName: "search", + type: "tool-call", + }, + ], + role: "assistant", + }, + ], }, - type: "session.started", - }, - { session: { id: "session-bounded-state" } }, - ); - - const state = fakeEve.values.get("braintrust.eve.tracing") as { - spanReferences: unknown[]; - stepStarts: unknown[]; - }; - expect(state.spanReferences).toHaveLength(10_000); - expect(state.stepStarts).toHaveLength(10_000); - }); - - it("does not emit a span for session lifecycle metadata alone", async () => { - const wildcard = braintrustEveHook({ defineState }).events?.["*"]; - - await wildcard?.( - { - data: { - runtime: { - agentId: "agent-session-only", - eveVersion: "0.20.0", - modelId: "openai/gpt-5.4-mini", - }, + model: { + modelId: "qwen/qwen3-30b-a3b", + provider: "openrouter.chat", }, - type: "session.started", + scope, + type: "model.call.started", }, - { session: { id: "session-only" } }, + secondModelContext, ); - - expect(await backgroundLogger.drain()).toEqual([]); - }); - - it("records a flat Eve turn with session model metadata", async () => { - const wildcard = braintrustEveHook({ - defineState, - metadata: { - scenario: "eve-plugin-unit", - testRunId: "test-run-flat-tree", - }, - }).events?.["*"]; - expect(wildcard).toBeDefined(); - - const ctx: EveHookContext = { - session: { id: "session-flat-tree" }, - }; - const emit = (event: EveHandleMessageStreamEvent) => wildcard?.(event, ctx); - const expectedModelMetadata = { - model: "braintrust-eve-mock", - provider: "eve-mock", - }; - - await emit({ - data: { - runtime: { - agentId: "agent-id", - agentName: "eve-test-agent", - eveVersion: "0.20.0", - modelId: "eve-mock/braintrust-eve-mock", - }, - }, - meta: { at: "2026-01-01T00:00:00.000Z" }, - type: "session.started", - }); - await emit({ - data: { sequence: 0, turnId: "turn-flat-tree" }, - meta: { at: "2026-01-01T00:00:00.010Z" }, - type: "turn.started", - }); - await emit({ - data: { - message: "Search then read", - sequence: 0, - turnId: "turn-flat-tree", - }, - meta: { at: "2026-01-01T00:00:00.020Z" }, - type: "message.received", - }); - await emit({ - data: { sequence: 0, stepIndex: 0, turnId: "turn-flat-tree" }, - meta: { at: "2026-01-01T00:00:00.030Z" }, - type: "step.started", - }); - await emit({ - data: { - actions: [ + await provider.events["model.call.completed"]( + { + content: [ { - callId: "call-search", - input: { query: "Eve instrumentation" }, - kind: "tool-call", - toolName: "search", + text: "Here is the Eve instrumentation guide.", + type: "text", }, ], - sequence: 0, - stepIndex: 0, - turnId: "turn-flat-tree", - }, - meta: { at: "2026-01-01T00:00:00.040Z" }, - type: "actions.requested", - }); - await emit({ - data: { - error: undefined, - result: { - callId: "call-search", - kind: "tool-result", - output: { hits: ["eve.dev/docs"] }, - toolName: "search", - }, - sequence: 0, - status: "completed", - stepIndex: 0, - turnId: "turn-flat-tree", - }, - meta: { at: "2026-01-01T00:00:00.050Z" }, - type: "action.result", - }); - await emit({ - data: { - finishReason: "tool-calls", - sequence: 0, - stepIndex: 0, - turnId: "turn-flat-tree", - usage: { - cacheReadTokens: 3, - cacheWriteTokens: 2, - costUsd: 0.001, - inputTokens: 10, - outputTokens: 5, - }, - }, - meta: { at: "2026-01-01T00:00:00.060Z" }, - type: "step.completed", - }); - await emit({ - data: { sequence: 0, stepIndex: 1, turnId: "turn-flat-tree" }, - meta: { at: "2026-01-01T00:00:00.070Z" }, - type: "step.started", - }); - await emit({ - data: { - finishReason: "stop", - message: "Here is the Eve instrumentation guide.", - sequence: 0, - stepIndex: 1, - turnId: "turn-flat-tree", - }, - meta: { at: "2026-01-01T00:00:00.080Z" }, - type: "message.completed", - }); - await emit({ - data: { finishReason: "stop", - sequence: 0, - stepIndex: 1, - turnId: "turn-flat-tree", + idempotencyKey: "model:attempt-1:0", + scope, + type: "model.call.completed", usage: { inputTokens: 20, outputTokens: 8, }, }, - meta: { at: "2026-01-01T00:00:00.090Z" }, - type: "step.completed", - }); - await emit({ - data: { sequence: 0, turnId: "turn-flat-tree" }, - meta: { at: "2026-01-01T00:00:00.100Z" }, - type: "turn.completed", - }); + secondModelContext, + ); + await provider.events["turn.completed"]( + { + idempotencyKey: "turn:session-root:turn-0", + sessionId: "session-root", + turnId: "turn-0", + type: "turn.completed", + }, + turnContext, + ); const spans = (await backgroundLogger.drain()) as Array< Record @@ -598,8 +257,8 @@ describe("braintrustEveHook", () => { const root = spans.find( (span) => span.span_attributes?.name === "eve.turn", ); - const steps = spans.filter((span) => - String(span.span_attributes?.name).startsWith("eve.step"), + const steps = spans.filter( + (span) => span.span_attributes?.name === "eve.step", ); const tool = spans.find((span) => span.span_attributes?.name === "search"); @@ -612,78 +271,58 @@ describe("braintrustEveHook", () => { expect(root).toMatchObject({ input: [{ content: "Search then read", role: "user" }], metadata: { - ...expectedModelMetadata, - "eve.session_id": "session-flat-tree", + "eve.session_id": "session-root", scenario: "eve-plugin-unit", testRunId: "test-run-flat-tree", }, + output: "Here is the Eve instrumentation guide.", metrics: { completion_tokens: 13, - estimated_cost: 0.001, - prompt_cached_tokens: 3, prompt_cache_creation_tokens: 2, + prompt_cached_tokens: 3, prompt_tokens: 30, tokens: 43, }, - output: "Here is the Eve instrumentation guide.", span_attributes: { name: "eve.turn", type: "task", }, - root_span_id: deterministicEveIdForTest( - "eve:root", - "session-flat-tree", - "turn-flat-tree", - ), span_parents: [], }); + expect(root?.root_span_id).not.toBe(root?.span_id); + expect(root?.metadata).not.toHaveProperty("model"); + expect(root?.metadata).not.toHaveProperty("provider"); + expect(steps).toHaveLength(2); - expect(steps.map((span) => span.span_attributes?.name)).toEqual([ - "eve.step", - "eve.step", - ]); - expect(steps.map((span) => span.span_attributes?.type)).toEqual([ - "llm", - "llm", - ]); - expect(steps.map((span) => span.span_parents)).toEqual([ - [root?.span_id], - [root?.span_id], - ]); for (const step of steps) { - expect(step.metadata).toEqual({ - ...expectedModelMetadata, - "eve.session_id": "session-flat-tree", - scenario: "eve-plugin-unit", - testRunId: "test-run-flat-tree", + expect(step).toMatchObject({ + metadata: { + "eve.session_id": "session-root", + model: "qwen/qwen3-30b-a3b", + provider: "openrouter", + scenario: "eve-plugin-unit", + testRunId: "test-run-flat-tree", + }, + span_attributes: { + name: "eve.step", + type: "llm", + }, + span_parents: [root?.span_id], }); } - expect(steps[0]?.input).toBeUndefined(); - expect(steps[1]?.input).toBeUndefined(); - expect(tool).toMatchObject({ - input: { query: "Eve instrumentation" }, - metadata: { - "eve.session_id": "session-flat-tree", - scenario: "eve-plugin-unit", - testRunId: "test-run-flat-tree", - }, - output: { hits: ["eve.dev/docs"] }, - span_attributes: { - name: "search", - type: "tool", - }, - span_parents: [root?.span_id], - }); - expect(tool?.metadata).not.toHaveProperty("model"); - expect(tool?.metadata).not.toHaveProperty("provider"); + expect(steps[0]?.metadata).not.toHaveProperty("tools"); expect(steps[0]?.output).toMatchObject([ { finish_reason: "tool_calls", message: { + content: null, + reasoning: [{ content: "I should search first." }], tool_calls: [ { function: { - arguments: JSON.stringify({ query: "Eve instrumentation" }), + arguments: JSON.stringify({ + query: "Eve instrumentation", + }), name: "search", }, id: "call-search", @@ -702,1570 +341,1004 @@ describe("braintrustEveHook", () => { }, }, ]); + expect(tool).toMatchObject({ + input: { query: "Eve instrumentation" }, + metadata: { + "eve.session_id": "session-root", + scenario: "eve-plugin-unit", + testRunId: "test-run-flat-tree", + }, + output: { hits: ["eve.dev/docs"] }, + span_attributes: { + name: "search", + type: "tool", + }, + span_parents: [root?.span_id], + }); + expect(tool?.metadata).not.toHaveProperty("model"); + expect(tool?.metadata).not.toHaveProperty("provider"); }); - it("records each user message as a separate turn in one session", async () => { - const wildcard = braintrustEveHook({ defineState }).events?.["*"]; - expect(wildcard).toBeDefined(); - - const ctx: EveHookContext = { session: { id: "session-multi-turn" } }; - for (const [sequence, message] of [ - [0, "First user message"], - [1, "Second user message"], - ] as const) { - const turnId = `turn-${sequence}`; - await wildcard?.( - { data: { sequence, turnId }, type: "turn.started" }, - ctx, - ); - await wildcard?.( - { data: { message, sequence, turnId }, type: "message.received" }, - ctx, - ); - await wildcard?.( - { data: { sequence, turnId }, type: "turn.completed" }, - ctx, - ); - } - - const spans = (await backgroundLogger.drain()) as Array< - Record - >; - const turns = spans.filter( - (span) => span.span_attributes?.name === "eve.turn", - ); + it("attaches a subagent turn beneath its action span", async () => { + const provider = braintrustEveInstrumentation(); + const parentTurnContext = providerContext(); + const actionContext = providerContext(); + const childTurnContext = providerContext(); + const childModelContext = providerContext(); + const parentScope = ATTEMPT_SCOPE; - expect( - spans.some((span) => span.span_attributes?.name === "eve.session"), - ).toBe(false); - expect(turns).toHaveLength(2); - expect(turns.map((turn) => turn.span_parents)).toEqual([[], []]); - expect(turns.map((turn) => turn.root_span_id)).toEqual([ - deterministicEveIdForTest("eve:root", "session-multi-turn", "turn-0"), - deterministicEveIdForTest("eve:root", "session-multi-turn", "turn-1"), - ]); - expect(turns[0]?.root_span_id).not.toBe(turns[1]?.root_span_id); - expect(turns.map((turn) => turn.input)).toEqual([ - [{ content: "First user message", role: "user" }], - [{ content: "Second user message", role: "user" }], - ]); - }); - - it("merges incremental tool-call batches without reconstructing later LLM inputs", async () => { - const wildcard = braintrustEveHook({ defineState }).events?.["*"]; - expect(wildcard).toBeDefined(); - - const ctx: EveHookContext = { - session: { id: "session-incremental-tools" }, - }; - const emit = (event: EveHandleMessageStreamEvent) => wildcard?.(event, ctx); - - await emit({ - data: { sequence: 0, turnId: "turn-incremental-tools" }, - type: "turn.started", - }); - await emit({ - data: { - message: "Search then read", - sequence: 0, - turnId: "turn-incremental-tools", - }, - type: "message.received", - }); - await emit({ - data: { sequence: 0, stepIndex: 0, turnId: "turn-incremental-tools" }, - type: "step.started", - }); - await emit({ - data: { - actions: [ - { - callId: "call-search", - input: { query: "Eve instrumentation" }, - kind: "tool-call", - toolName: "search", - }, - ], + await provider.events["turn.started"]( + { + idempotencyKey: "turn:session-root:turn-0", + rootSessionId: "session-root", sequence: 0, - stepIndex: 0, - turnId: "turn-incremental-tools", + sessionId: "session-root", + turnId: "turn-0", + type: "turn.started", }, - type: "actions.requested", - }); - await emit({ - data: { - actions: [ - { - callId: "call-search", - input: { query: "Updated Eve instrumentation" }, - kind: "tool-call", - toolName: "search", - }, - ], - sequence: 0, - stepIndex: 0, - turnId: "turn-incremental-tools", + parentTurnContext, + ); + await provider.events["action.started"]( + { + callId: "call-researcher", + idempotencyKey: "action:session-root:turn-0:call-researcher", + input: { message: "Research Eve" }, + name: "researcher", + scope: parentScope, + type: "action.started", }, - type: "actions.requested", - }); - await emit({ - data: { - actions: [ - { - callId: "call-read", - input: { url: "https://eve.dev/docs/guides/instrumentation" }, - kind: "tool-call", - toolName: "read", + actionContext, + ); + await Promise.all([ + provider.events["turn.started"]( + { + idempotencyKey: "turn:session-child:turn-0", + parentLineage: { + callId: "call-researcher", + sessionId: "session-root", + turnId: "turn-0", }, - ], - sequence: 0, - stepIndex: 0, - turnId: "turn-incremental-tools", - }, - type: "actions.requested", - }); - await emit({ - data: { - result: { - callId: "call-search", - kind: "tool-result", - output: { url: "https://eve.dev/docs/guides/instrumentation" }, - toolName: "search", + rootSessionId: "session-root", + sequence: 0, + sessionId: "session-child", + turnId: "turn-0", + type: "turn.started", }, - sequence: 0, - status: "completed", - stepIndex: 0, - turnId: "turn-incremental-tools", - }, - type: "action.result", - }); - await emit({ - data: { - result: { - callId: "call-read", - kind: "tool-result", - output: { excerpt: "Eve hooks expose runtime stream events." }, - toolName: "read", + childTurnContext, + ), + provider.events["model.call.started"]( + { + idempotencyKey: "model:child-attempt-0:0", + model: { + modelId: "qwen/qwen3-30b-a3b", + provider: "openrouter.chat", + }, + scope: { + attemptId: "child-attempt-0", + attemptIndex: 0, + rootSessionId: "session-root", + sessionId: "session-child", + stepIndex: 0, + turnId: "turn-0", + }, + type: "model.call.started", }, - sequence: 0, - status: "completed", - stepIndex: 0, - turnId: "turn-incremental-tools", - }, - type: "action.result", - }); - await emit({ - data: { - finishReason: "tool-calls", - sequence: 0, - stepIndex: 0, - turnId: "turn-incremental-tools", - }, - type: "step.completed", - }); - await emit({ - data: { sequence: 0, stepIndex: 1, turnId: "turn-incremental-tools" }, - type: "step.started", - }); - await emit({ - data: { - finishReason: "stop", - message: "Done.", - sequence: 0, - stepIndex: 1, - turnId: "turn-incremental-tools", - }, - type: "message.completed", - }); - await emit({ - data: { - finishReason: "stop", - sequence: 0, - stepIndex: 1, - turnId: "turn-incremental-tools", - }, - type: "step.completed", - }); - await emit({ - data: { sequence: 0, turnId: "turn-incremental-tools" }, - type: "turn.completed", - }); + childModelContext, + ), + ]); const spans = (await backgroundLogger.drain()) as Array< Record >; - const steps = spans.filter((span) => - String(span.span_attributes?.name).startsWith("eve.step"), + const parent = spans.find( + (span) => + span.span_attributes?.name === "eve.turn" && + span.metadata?.["eve.session_id"] === "session-root", + ); + const action = spans.find( + (span) => span.span_attributes?.name === "researcher", + ); + const child = spans.find( + (span) => + span.span_attributes?.name === "eve.turn" && + span.metadata?.["eve.session_id"] === "session-child", + ); + const childModel = spans.find( + (span) => span.span_attributes?.name === "eve.step", ); - expect(steps[0]?.output).toMatchObject([ - { - finish_reason: "tool_calls", - message: { - tool_calls: [ - { - function: { - arguments: JSON.stringify({ - query: "Updated Eve instrumentation", - }), - name: "search", - }, - id: "call-search", - type: "function", - }, - { - function: { - arguments: JSON.stringify({ - url: "https://eve.dev/docs/guides/instrumentation", - }), - name: "read", - }, - id: "call-read", - type: "function", - }, - ], - }, - }, - ]); - expect(steps[1]?.input).toBeUndefined(); + expect(action?.span_parents).toEqual([parent?.span_id]); + expect(child?.span_parents).toEqual([action?.span_id]); + expect(child?.root_span_id).toBe(parent?.root_span_id); + expect(childModel?.span_parents).toEqual([child?.span_id]); + expect(childModel?.root_span_id).toBe(parent?.root_span_id); }); - it("preserves reasoning across tool, text, and structured outputs", async () => { - const wildcard = braintrustEveHook({ defineState }).events?.["*"]; - const ctx: EveHookContext = { - session: { id: "session-reasoning-outputs" }, + it("keeps a stable root across nested subagents with different turn ids", async () => { + const provider = braintrustEveInstrumentation(); + const rootContext = providerContext(); + const rootScope = { ...ATTEMPT_SCOPE, turnId: "root-turn" }; + const childScope: EveInstrumentationAttemptScope = { + attemptId: "child-attempt", + attemptIndex: 0, + rootSessionId: "session-root", + sessionId: "session-child", + stepIndex: 0, + turnId: "child-turn", }; - const emit = (event: EveHandleMessageStreamEvent) => wildcard?.(event, ctx); - const turnId = "turn-reasoning-outputs"; - - await emit({ data: { sequence: 0, turnId }, type: "turn.started" }); - await emit({ - data: { sequence: 0, stepIndex: 0, turnId }, - type: "step.started", - }); - await emit({ - data: { - reasoning: "I should inspect the available tools.", - sequence: 0, - stepIndex: 0, - turnId, - }, - meta: { at: "2026-01-01T00:00:00.010Z" }, - type: "reasoning.completed", - }); - await emit({ - data: { - finishReason: "tool-calls", - message: null, - sequence: 0, - stepIndex: 0, - turnId, - }, - type: "message.completed", - }); - await emit({ - data: { - actions: [ - { - callId: "call-reasoning-search", - input: { query: "Eve reasoning" }, - kind: "tool-call", - toolName: "search", - }, - ], - sequence: 0, - stepIndex: 0, - turnId, - }, - type: "actions.requested", - }); - await emit({ - data: { finishReason: "tool-calls", sequence: 0, stepIndex: 0, turnId }, - type: "step.completed", - }); - - await emit({ - data: { sequence: 0, stepIndex: 1, turnId }, - type: "step.started", - }); - await emit({ - data: { - reasoning: "The tool returned the relevant guide.", - sequence: 0, - stepIndex: 1, - turnId, - }, - meta: { at: "2026-01-01T00:00:00.020Z" }, - type: "reasoning.completed", - }); - await emit({ - data: { - reasoning: "I can now answer concisely.", - sequence: 0, - stepIndex: 1, - turnId, - }, - meta: { at: "2026-01-01T00:00:00.030Z" }, - type: "reasoning.completed", - }); - await emit({ - data: { - finishReason: "stop", - message: "Eve exposes completed reasoning events.", - sequence: 0, - stepIndex: 1, - turnId, - }, - type: "message.completed", - }); - await emit({ - data: { finishReason: "stop", sequence: 0, stepIndex: 1, turnId }, - type: "step.completed", - }); - - await emit({ - data: { sequence: 0, stepIndex: 2, turnId }, - type: "step.started", - }); - await emit({ - data: { - reasoning: "The response must match the requested schema.", - sequence: 0, - stepIndex: 2, - turnId, + await provider.events["turn.started"]( + { + idempotencyKey: "turn:session-root:root-turn", + rootSessionId: "session-root", + sequence: 4, + sessionId: "session-root", + turnId: "root-turn", + type: "turn.started", }, - meta: { at: "2026-01-01T00:00:00.040Z" }, - type: "reasoning.completed", - }); - await emit({ - data: { - result: { supported: true }, - sequence: 0, - stepIndex: 2, - turnId, + rootContext, + ); + await provider.events["action.started"]( + { + callId: "call-child", + idempotencyKey: "action:session-root:root-turn:call-child", + name: "child", + scope: rootScope, + type: "action.started", }, - type: "result.completed", - }); - await emit({ - data: { finishReason: "stop", sequence: 0, stepIndex: 2, turnId }, - type: "step.completed", - }); - await emit({ data: { sequence: 0, turnId }, type: "turn.completed" }); - - const spans = (await backgroundLogger.drain()) as Array< - Record - >; - const steps = spans.filter( - (span) => span.span_attributes?.name === "eve.step", + providerContext(), ); - expect(steps).toHaveLength(3); - expect(steps[0]?.output).toMatchObject([ + await provider.events["turn.started"]( { - finish_reason: "tool_calls", - message: { - content: null, - reasoning: [{ content: "I should inspect the available tools." }], - tool_calls: [ - { - function: { name: "search" }, - id: "call-reasoning-search", - type: "function", - }, - ], + idempotencyKey: "turn:session-child:child-turn", + parentLineage: { + callId: "call-child", + sessionId: "session-root", + turnId: "root-turn", }, + rootSessionId: "session-root", + sequence: 2, + sessionId: "session-child", + turnId: "child-turn", + type: "turn.started", }, - ]); - expect(steps[1]?.output).toMatchObject([ + providerContext(), + ); + await provider.events["action.started"]( { - finish_reason: "stop", - message: { - content: "Eve exposes completed reasoning events.", - reasoning: [ - { content: "The tool returned the relevant guide." }, - { content: "I can now answer concisely." }, - ], - }, + callId: "call-grandchild", + idempotencyKey: "action:session-child:child-turn:call-grandchild", + name: "grandchild", + scope: childScope, + type: "action.started", }, - ]); - expect(steps[2]?.output).toMatchObject([ + providerContext(), + ); + const rootReference = rootContext.state.get(); + if ( + typeof rootReference !== "object" || + rootReference === null || + Array.isArray(rootReference) || + typeof rootReference.rootSpanId !== "string" + ) { + throw new Error("Expected the root turn trace context to be persisted"); + } + const resumedProvider = braintrustEveInstrumentation(); + await resumedProvider.events["turn.started"]( { - finish_reason: "stop", - message: { - content: { supported: true }, - reasoning: [ - { content: "The response must match the requested schema." }, - ], + idempotencyKey: "turn:session-grandchild:grandchild-turn", + parentLineage: { + callId: "call-grandchild", + sessionId: "session-child", + turnId: "child-turn", + }, + parentTraceContext: { + spanId: "1111111111111111", + traceFlags: 1, + traceId: rootReference.rootSpanId, }, + rootSessionId: "session-root", + sequence: 7, + sessionId: "session-grandchild", + turnId: "grandchild-turn", + type: "turn.started", }, + providerContext(), + ); + + const spans = (await backgroundLogger.drain()) as Array< + Record + >; + const turns = spans.filter( + (span) => span.span_attributes?.name === "eve.turn", + ); + const root = turns.find( + (turn) => turn.metadata?.["eve.session_id"] === "session-root", + ); + const child = turns.find( + (turn) => turn.metadata?.["eve.session_id"] === "session-child", + ); + const grandchild = turns.find( + (turn) => turn.metadata?.["eve.session_id"] === "session-grandchild", + ); + const childAction = spans.find( + (span) => span.span_attributes?.name === "child", + ); + const grandchildAction = spans.find( + (span) => span.span_attributes?.name === "grandchild", + ); + + expect(child?.span_parents).toEqual([childAction?.span_id]); + expect(grandchild?.span_parents).toEqual([grandchildAction?.span_id]); + expect(turns.map((turn) => turn.root_span_id)).toEqual([ + root?.root_span_id, + root?.root_span_id, + root?.root_span_id, ]); }); - it("rehydrates reasoning idempotently across Vercel Workflow steps", async () => { - const eveState = createFakeDefineState(); - const ctx: EveHookContext = { - session: { id: "session-reasoning-replay" }, - }; - const turnId = "turn-reasoning-replay"; - const reasoningEvent = { - data: { - reasoning: "Use the search tool before answering.", - sequence: 0, - stepIndex: 0, - turnId, - }, - meta: { at: "2026-01-01T00:00:00.020Z" }, - type: "reasoning.completed", - } as const satisfies EveHandleMessageStreamEvent; - const initialHook = braintrustEveHook({ - defineState: eveState.defineState, - }).events?.["*"]; + it("creates a separate trace for each top-level turn in one session", async () => { + const provider = braintrustEveInstrumentation(); - await initialHook?.( + await provider.events["turn.started"]( { - data: { sequence: 0, turnId }, - meta: { at: "2026-01-01T00:00:00.000Z" }, + idempotencyKey: "turn:shared-session:turn-0", + rootSessionId: "shared-session", + sequence: 0, + sessionId: "shared-session", + turnId: "turn-0", type: "turn.started", }, - ctx, + providerContext(), ); - await initialHook?.( + await provider.events["turn.started"]( { - data: { sequence: 0, stepIndex: 0, turnId }, - meta: { at: "2026-01-01T00:00:00.010Z" }, - type: "step.started", + idempotencyKey: "turn:shared-session:turn-1", + rootSessionId: "shared-session", + sequence: 1, + sessionId: "shared-session", + turnId: "turn-1", + type: "turn.started", }, - ctx, + providerContext(), ); - await initialHook?.(reasoningEvent, ctx); - const initialWrites = (await backgroundLogger.drain()) as Array< - Record & { id: string } + const turns = (await backgroundLogger.drain()) as Array< + Record >; - expect( - initialWrites - .filter((span) => span.span_attributes?.name) - .every( - (span) => span.context?.span_origin?.instrumentation?.name === "eve", - ), - ).toBe(true); - for (const [key, value] of eveState.values) { - eveState.values.set(key, JSON.parse(JSON.stringify(value))); - } + expect(turns).toHaveLength(2); + expect(turns.map((turn) => turn.metadata?.["eve.session_id"])).toEqual([ + "shared-session", + "shared-session", + ]); + expect(new Set(turns.map((turn) => turn.root_span_id)).size).toBe(2); + }); + + it("preserves a durable subagent turn across provider instances", async () => { + const firstProvider = braintrustEveInstrumentation(); + const parentTurnContext = providerContext(); + const childTurnContext = providerContext(); - const resumedHook = braintrustEveHook({ - defineState: eveState.defineState, - }).events?.["*"]; - await resumedHook?.( + await firstProvider.events["turn.started"]( { - data: { sequence: 0, turnId }, - meta: { at: "2026-01-01T00:00:00.000Z" }, + idempotencyKey: "turn:session-root:turn-0", + rootSessionId: "session-root", + sequence: 0, + sessionId: "session-root", + turnId: "turn-0", type: "turn.started", }, - ctx, + parentTurnContext, ); - await resumedHook?.( + await firstProvider.events["action.started"]( { - data: { sequence: 0, stepIndex: 0, turnId }, - meta: { at: "2026-01-01T00:00:00.010Z" }, - type: "step.started", + callId: "call-researcher", + idempotencyKey: "action:session-root:turn-0:call-researcher", + name: "researcher", + scope: ATTEMPT_SCOPE, + type: "action.started", }, - ctx, + providerContext(), ); - const resumedStartWrites = (await backgroundLogger.drain()) as Array< - Record & { id: string } - >; - await resumedHook?.(reasoningEvent, ctx); - await resumedHook?.( + await firstProvider.events["turn.started"]( { - data: { - finishReason: "tool-calls", - message: null, - sequence: 0, - stepIndex: 0, - turnId, + idempotencyKey: "turn:session-child:turn-0", + parentLineage: { + callId: "call-researcher", + sessionId: "session-root", + turnId: "turn-0", }, - type: "message.completed", + rootSessionId: "session-root", + sequence: 0, + sessionId: "session-child", + turnId: "turn-0", + type: "turn.started", }, - ctx, + childTurnContext, ); - await resumedHook?.( + + const resumedProvider = braintrustEveInstrumentation(); + const childScope: EveInstrumentationAttemptScope = { + attemptId: "child-attempt-0", + attemptIndex: 0, + rootSessionId: "session-root", + sessionId: "session-child", + stepIndex: 0, + turnId: "turn-0", + }; + const modelContext = providerContext(); + await resumedProvider.events["model.call.started"]( { - data: { - actions: [ - { - callId: "call-replayed-search", - input: { query: "Vercel Workflow reasoning" }, - kind: "tool-call", - toolName: "search", - }, - ], - sequence: 0, - stepIndex: 0, - turnId, + idempotencyKey: "model:child-attempt-0:0", + input: { + messages: [{ content: "Continue durably", role: "user" }], }, - type: "actions.requested", + model: { modelId: "gpt-5.4-mini", provider: "openai.responses" }, + scope: childScope, + type: "model.call.started", }, - ctx, + modelContext, ); - const resumedIntermediateWrites = (await backgroundLogger.drain()) as Array< - Record & { id: string } - >; - await resumedHook?.( + await resumedProvider.events["model.call.completed"]( { - data: { finishReason: "tool-calls", sequence: 0, stepIndex: 0, turnId }, - type: "step.completed", + content: [{ text: "Durable answer", type: "text" }], + finishReason: "stop", + idempotencyKey: "model:child-attempt-0:0", + scope: childScope, + type: "model.call.completed", + usage: { inputTokens: 3, outputTokens: 2 }, }, - ctx, + modelContext, ); - await resumedHook?.( - { data: { sequence: 0, turnId }, type: "turn.completed" }, - ctx, + await resumedProvider.events["turn.completed"]( + { + idempotencyKey: "turn:session-child:turn-0", + sessionId: "session-child", + turnId: "turn-0", + type: "turn.completed", + }, + providerContext(childTurnContext.state.get()), ); - const resumedWrites = (await backgroundLogger.drain()) as Array< - Record & { id: string } + const spans = (await backgroundLogger.drain()) as Array< + Record >; - const allWrites = [ - ...initialWrites, - ...resumedStartWrites, - ...resumedIntermediateWrites, - ...resumedWrites, - ]; - const stepRowId = deterministicEveIdForTest( - "eve:row:step", - "session-reasoning-replay", - turnId, - "0", + const parent = spans.find( + (span) => + span.span_attributes?.name === "eve.turn" && + span.metadata?.["eve.session_id"] === "session-root", ); - expect( - allWrites.filter( - (write) => write.id === stepRowId && write.output !== undefined, - ), - ).toHaveLength(1); - - const spans = mergeRowBatch([...allWrites].reverse()); - const steps = spans.filter( - (span) => span.span_attributes?.name === "eve.step", + const action = spans.find( + (span) => span.span_attributes?.name === "researcher", ); - expect(steps).toHaveLength(1); - expect(steps[0]?.span_id).toBe( - deterministicEveIdForTest( - "eve:step", - "session-reasoning-replay", - turnId, - "0", - ), + const child = spans.find( + (span) => + span.span_attributes?.name === "eve.turn" && + span.metadata?.["eve.session_id"] === "session-child", ); - expect(steps[0]?.output?.[0]?.message).toMatchObject({ - content: null, - reasoning: [{ content: "Use the search tool before answering." }], - tool_calls: [ - { - function: { name: "search" }, - id: "call-replayed-search", - type: "function", - }, - ], - }); - expect(steps[0]?.output?.[0]?.message?.reasoning).toHaveLength(1); - expect(eveState.values.get("braintrust.eve.tracing")).toMatchObject({ - reasoningBlocks: [], - stepStarts: [], + const step = spans.find( + (span) => span.span_attributes?.name === "eve.step", + ); + expect(child).toMatchObject({ + input: [{ content: "Continue durably", role: "user" }], + metrics: { + completion_tokens: 2, + prompt_tokens: 3, + tokens: 5, + }, + output: "Durable answer", + span_parents: [action?.span_id], }); + expect(child?.root_span_id).toBe(parent?.root_span_id); + expect(step?.span_parents).toEqual([child?.span_id]); + expect(step?.root_span_id).toBe(parent?.root_span_id); }); - it("merges late tool results into tool spans closed by turn completion", async () => { - const eveState = createFakeDefineState(); - const wildcard = braintrustEveHook({ - defineState: eveState.defineState, - metadata: { - scenario: "eve-plugin-unit", - testRunId: "test-run-late-tool-result", - }, - }).events?.["*"]; - expect(wildcard).toBeDefined(); + it("resumes spans and turn metrics from Eve operation state", async () => { + const firstProvider = braintrustEveInstrumentation(); + const turnContext = providerContext(); + const modelContext = providerContext(); + const scope = ATTEMPT_SCOPE; - const ctx: EveHookContext = { - session: { id: "session-late-tool-result" }, - }; - const emit = (event: EveHandleMessageStreamEvent) => wildcard?.(event, ctx); - - await emit({ - data: { sequence: 0, turnId: "turn-late-tool-result" }, - meta: { at: "2026-01-01T00:00:00.000Z" }, - type: "turn.started", - }); - await emit({ - data: { sequence: 0, stepIndex: 0, turnId: "turn-late-tool-result" }, - meta: { at: "2026-01-01T00:00:00.010Z" }, - type: "step.started", - }); - await emit({ - data: { - actions: [ - { - callId: "call-late-search", - input: { query: "late tool result" }, - kind: "tool-call", - toolName: "search", - }, - ], + await firstProvider.events["turn.started"]( + { + idempotencyKey: "turn:session-root:turn-0", + rootSessionId: "session-root", sequence: 0, - stepIndex: 0, - turnId: "turn-late-tool-result", - }, - meta: { at: "2026-01-01T00:00:00.020Z" }, - type: "actions.requested", - }); - await emit({ - data: { sequence: 0, turnId: "turn-late-tool-result" }, - meta: { at: "2026-01-01T00:00:00.030Z" }, - type: "turn.completed", - }); - expect(eveState.values.get("braintrust.eve.tracing")).toMatchObject({ - spanReferences: expect.arrayContaining([ - expect.objectContaining({ - exported: expect.any(String), - rootSpanId: deterministicEveIdForTest( - "eve:root", - "session-late-tool-result", - "turn-late-tool-result", - ), - rowId: deterministicEveIdForTest( - "eve:row:tool", - "session-late-tool-result", - "turn-late-tool-result", - "call-late-search", - ), - spanId: deterministicEveIdForTest( - "eve:tool", - "session-late-tool-result", - "turn-late-tool-result", - "call-late-search", - ), - }), - ]), - stepStarts: [], - }); - const initialWrites = (await backgroundLogger.drain()) as Array< - Record & { id: string } - >; - const resumedWildcard = braintrustEveHook({ - defineState: eveState.defineState, - metadata: { - scenario: "eve-plugin-unit", - testRunId: "test-run-late-tool-result", + sessionId: "session-root", + turnId: "turn-0", + type: "turn.started", }, - }).events?.["*"]; - const flushSpy = vi.spyOn(backgroundLogger, "flush"); - flushSpy.mockClear(); - await resumedWildcard?.( + turnContext, + ); + await firstProvider.events["model.call.started"]( { - data: { - result: { - callId: "call-late-search", - kind: "tool-result", - output: { title: "Late result" }, - toolName: "search", - }, - sequence: 0, - status: "completed", - stepIndex: 0, - turnId: "turn-late-tool-result", + idempotencyKey: "model:attempt-0:0", + input: { + messages: [{ content: "Resume this call", role: "user" }], + }, + model: { + modelId: "gpt-5.4-mini", + provider: "openai.responses", }, - meta: { at: "2026-01-01T00:00:00.040Z" }, - type: "action.result", + scope, + type: "model.call.started", }, - ctx, - ); - expect(flushSpy).not.toHaveBeenCalled(); - - const resumedWrites = (await backgroundLogger.drain()) as Array< - Record & { id: string } - >; - expect( - [...initialWrites, ...resumedWrites].every( - (span) => span._is_merge === true, - ), - ).toBe(true); - - // The backend may ingest separate workflow uploads out of order. Because - // every write is a merge, a delayed initial write cannot erase the result. - const spans = mergeRowBatch([...resumedWrites, ...initialWrites]); - const turns = spans.filter( - (span) => span.span_attributes?.name === "eve.turn", + modelContext, ); - const tool = spans.find((span) => span.span_attributes?.name === "search"); - expect(turns).toHaveLength(1); - expect( - spans.filter((span) => span.span_attributes?.name === "search"), - ).toHaveLength(1); - expect(tool).toMatchObject({ - input: { query: "late tool result" }, - metadata: { - scenario: "eve-plugin-unit", - testRunId: "test-run-late-tool-result", - }, - output: { title: "Late result" }, - span_attributes: { - name: "search", - type: "tool", - }, - span_parents: [turns[0]?.span_id], + const persisted = modelContext.state.get(); + expect(persisted).toMatchObject({ + exported: expect.any(String), + rootSpanId: expect.any(String), + spanId: expect.any(String), }); - expect(tool?.metrics?.end).toEqual(expect.any(Number)); - }); - it("lets action results complete sparse subagent events across workflow steps", async () => { - const eveState = createFakeDefineState(); - const ctx: EveHookContext = { - session: { id: "session-sparse-subagent" }, - }; - const firstWildcard = braintrustEveHook({ - defineState: eveState.defineState, - }).events?.["*"]; - - await firstWildcard?.( + const resumedProvider = braintrustEveInstrumentation(); + await resumedProvider.events["model.call.completed"]( { - data: { sequence: 0, turnId: "turn-sparse-subagent" }, - type: "turn.started", + content: [{ text: "Resumed output", type: "text" }], + finishReason: "stop", + idempotencyKey: "model:attempt-0:0", + scope, + type: "model.call.completed", + usage: { inputTokens: 4, outputTokens: 2 }, }, - ctx, + providerContext(persisted), ); - await firstWildcard?.( + const secondModelContext = providerContext(); + await resumedProvider.events["model.call.started"]( { - data: { - actions: [ - { - callId: "call-sparse-subagent", - input: { message: "Research Eve" }, - kind: "subagent-call", - subagentName: "researcher", - }, - ], - sequence: 0, - stepIndex: 0, - turnId: "turn-sparse-subagent", + idempotencyKey: "model:attempt-0:1", + input: { + messages: [{ content: "Continue this call", role: "user" }], + }, + model: { + modelId: "gpt-5.4-mini", + provider: "openai.responses", }, - type: "actions.requested", + scope: { ...scope, stepIndex: 1 }, + type: "model.call.started", }, - ctx, + secondModelContext, ); - await firstWildcard?.( - { - data: { - callId: "call-sparse-subagent", - sequence: 0, - subagentName: "researcher", - turnId: "turn-sparse-subagent", + const secondPersisted = secondModelContext.state.get(); + expect(secondPersisted).toMatchObject({ + turnMetricContributions: { + "model:attempt-0:0": { + completion_tokens: 2, + prompt_tokens: 4, + tokens: 6, }, - meta: { at: "2026-01-01T00:00:00.100Z" }, - type: "subagent.completed", }, - ctx, - ); - - const resumedWildcard = braintrustEveHook({ - defineState: eveState.defineState, - }).events?.["*"]; - await resumedWildcard?.( + }); + const finalProvider = braintrustEveInstrumentation(); + await finalProvider.events["model.call.completed"]( { - data: { - result: { - callId: "call-sparse-subagent", - kind: "subagent-result", - output: { answer: "Authoritative result" }, - subagentName: "researcher", - }, - sequence: 0, - status: "completed", - stepIndex: 0, - turnId: "turn-sparse-subagent", - }, - meta: { at: "2026-01-01T00:00:00.500Z" }, - type: "action.result", + content: [{ text: "Final resumed output", type: "text" }], + finishReason: "stop", + idempotencyKey: "model:attempt-0:1", + scope: { ...scope, stepIndex: 1 }, + type: "model.call.completed", + usage: { inputTokens: 3, outputTokens: 1 }, }, - ctx, + providerContext(secondPersisted), ); const spans = (await backgroundLogger.drain()) as Array< Record >; - const subagents = spans.filter( - (span) => span.span_attributes?.name === "researcher", + const steps = spans.filter( + (span) => span.span_attributes?.name === "eve.step", ); - expect(subagents).toHaveLength(1); - expect(subagents[0]).toMatchObject({ - output: { answer: "Authoritative result" }, - metrics: { end: Date.parse("2026-01-01T00:00:00.100Z") / 1000 }, - span_attributes: { type: "tool" }, - }); - }); - - it("evicts tracing state after session completion", async () => { - const fakeEve = createFakeDefineState(); - defineState = fakeEve.defineState; - const wildcard = braintrustEveHook({ - defineState, - metadata: { - scenario: "eve-plugin-unit", - testRunId: "test-run-late-after-session", + expect(steps).toHaveLength(2); + expect(steps[0]).toMatchObject({ + input: [{ content: "Resume this call", role: "user" }], + metrics: { + completion_tokens: 2, + prompt_tokens: 4, + tokens: 6, }, - }).events?.["*"]; - expect(wildcard).toBeDefined(); - - const ctx: EveHookContext = { - session: { id: "session-late-after-session" }, - }; - const emit = (event: EveHandleMessageStreamEvent) => wildcard?.(event, ctx); - - await emit({ - data: { sequence: 0, turnId: "turn-late-after-session" }, - meta: { at: "2026-01-01T00:00:00.000Z" }, - type: "turn.started", - }); - await emit({ - data: { sequence: 0, stepIndex: 0, turnId: "turn-late-after-session" }, - meta: { at: "2026-01-01T00:00:00.010Z" }, - type: "step.started", - }); - await emit({ - data: { - actions: [ - { - callId: "call-after-session", - input: { query: "after session" }, - kind: "tool-call", - toolName: "search", + output: [ + { + finish_reason: "stop", + message: { + content: "Resumed output", + role: "assistant", }, - ], - sequence: 0, - stepIndex: 0, - turnId: "turn-late-after-session", - }, - meta: { at: "2026-01-01T00:00:00.020Z" }, - type: "actions.requested", - }); - await emit({ - meta: { at: "2026-01-01T00:00:00.030Z" }, - type: "session.completed", - }); - - const spans = (await backgroundLogger.drain()) as Array< - Record - >; - const tool = spans.find((span) => span.span_attributes?.name === "search"); - expect(tool).toMatchObject({ - input: { query: "after session" }, - }); - expect(tool?.metrics?.end).toEqual(expect.any(Number)); - expect(fakeEve.values.get("braintrust.eve.tracing")).toEqual({ - llmInputs: [], - metadata: {}, - reasoningBlocks: [], - spanReferences: [], - stepStarts: [], - }); - }); - - it("records result-only tool events without invented input", async () => { - const wildcard = braintrustEveHook({ - defineState, - metadata: { - scenario: "eve-plugin-unit", - testRunId: "test-run-result-only", - }, - }).events?.["*"]; - expect(wildcard).toBeDefined(); - - const ctx: EveHookContext = { - session: { id: "session-result-only" }, - }; - const emit = (event: EveHandleMessageStreamEvent) => wildcard?.(event, ctx); - - await emit({ - data: { - result: { - callId: "call-result-only", - kind: "tool-result", - output: { title: "Result only" }, - toolName: "search", }, - sequence: 0, - status: "completed", - stepIndex: 0, - turnId: "turn-result-only", - }, - type: "action.result", - }); - await emit({ - data: { sequence: 0, turnId: "turn-result-only" }, - type: "turn.completed", + ], }); - - const spans = (await backgroundLogger.drain()) as Array< - Record - >; - const tool = spans.find((span) => span.span_attributes?.name === "search"); - expect(tool).toMatchObject({ - output: { title: "Result only" }, - span_attributes: { name: "search", type: "tool" }, + const turn = spans.find( + (span) => span.span_attributes?.name === "eve.turn", + ); + expect(turn?.metrics).toMatchObject({ + completion_tokens: 3, + prompt_tokens: 7, + tokens: 10, }); - expect(tool?.input).toBeUndefined(); }); - it("evicts tracing state after session failure", async () => { - const fakeEve = createFakeDefineState(); - const wildcard = braintrustEveHook({ - defineState: fakeEve.defineState, - }).events?.["*"]; - const ctx: EveHookContext = { - session: { id: "session-failed-cleanup" }, - }; - - await wildcard?.( + it("does not double-count replayed model completions", async () => { + const provider = braintrustEveInstrumentation(); + const modelContext = providerContext(); + await provider.events["turn.started"]( { - data: { sequence: 0, turnId: "turn-failed-cleanup" }, + idempotencyKey: "turn:replay-session:turn-0", + rootSessionId: "replay-session", + sequence: 0, + sessionId: "replay-session", + turnId: "turn-0", type: "turn.started", }, - ctx, - ); - await wildcard?.( - { - data: { - sequence: 0, - stepIndex: 0, - turnId: "turn-failed-cleanup", - }, - type: "step.started", - }, - ctx, + providerContext(), ); - await wildcard?.( + const scope = { + ...ATTEMPT_SCOPE, + attemptId: "replay-attempt", + rootSessionId: "replay-session", + sessionId: "replay-session", + }; + await provider.events["model.call.started"]( { - data: { - code: "session_failed", - message: "Session failed", - sessionId: "session-failed-cleanup", - }, - type: "session.failed", + idempotencyKey: "model:replay-attempt:0", + model: { modelId: "gpt-5.4-mini", provider: "openai.responses" }, + scope, + type: "model.call.started", }, - ctx, + modelContext, ); + const completed: EveInstrumentationModelCallCompletedEvent = { + content: [{ text: "Done", type: "text" }], + finishReason: "stop", + idempotencyKey: "model:replay-attempt:0", + scope, + type: "model.call.completed", + usage: { inputTokens: 3, outputTokens: 2 }, + }; + await provider.events["model.call.completed"](completed, modelContext); + await provider.events["model.call.completed"](completed, modelContext); - expect(fakeEve.values.get("braintrust.eve.tracing")).toEqual({ - llmInputs: [], - metadata: {}, - reasoningBlocks: [], - spanReferences: [], - stepStarts: [], + const spans = (await backgroundLogger.drain()) as Array< + Record + >; + const turn = spans.find( + (span) => span.span_attributes?.name === "eve.turn", + ); + expect(turn?.metrics).toMatchObject({ + completion_tokens: 2, + prompt_tokens: 3, + tokens: 5, }); }); - it("flushes final session events but not ordinary or ignored events", async () => { - const wildcard = braintrustEveHook({ defineState }).events?.["*"]; - const ctx: EveHookContext = { - session: { id: "session-selective-flush" }, - }; + it("records model, action, and turn failures as errors", async () => { + const provider = braintrustEveInstrumentation(); + const turnContext = providerContext(); + const modelContext = providerContext(); + const actionContext = providerContext(); + const scope = ATTEMPT_SCOPE; - await wildcard?.( + await provider.events["turn.started"]( { - data: { sequence: 0, turnId: "turn-selective-flush" }, + idempotencyKey: "turn:session-root:turn-0", + rootSessionId: "session-root", + sequence: 0, + sessionId: "session-root", + turnId: "turn-0", type: "turn.started", }, - ctx, + turnContext, ); - const flushSpy = vi - .spyOn(backgroundLogger, "flush") - .mockResolvedValue(undefined); - - await wildcard?.( + await provider.events["model.call.started"]( { - data: { wait: "next-user-message" }, - type: "session.waiting", + idempotencyKey: "model:attempt-0:0", + model: { modelId: "gpt-5.4-mini", provider: "openai.responses" }, + scope, + type: "model.call.started", }, - ctx, + modelContext, ); - expect(flushSpy).not.toHaveBeenCalled(); - - await wildcard?.( + await provider.events["model.call.failed"]( { - data: { - finishReason: "stop", - message: null, - sequence: 0, - stepIndex: 0, - turnId: "turn-selective-flush", - }, - type: "message.completed", + error: new Error("model exploded"), + idempotencyKey: "model:attempt-0:0", + scope, + type: "model.call.failed", }, - ctx, + modelContext, ); - expect(flushSpy).not.toHaveBeenCalled(); - - await wildcard?.( + await provider.events["action.started"]( { - type: "session.completed", + callId: "call-failing", + idempotencyKey: "action:session-root:turn-0:call-failing", + name: "failing-tool", + scope, + type: "action.started", }, - ctx, + actionContext, + ); + await provider.events["action.failed"]( + { + error: new Error("tool exploded"), + errorCode: "TOOL_FAILED", + idempotencyKey: "action:session-root:turn-0:call-failing", + outcome: "failed", + scope, + type: "action.failed", + }, + actionContext, + ); + await provider.events["turn.failed"]( + { + error: new Error("turn exploded"), + idempotencyKey: "turn:session-root:turn-0", + sessionId: "session-root", + turnId: "turn-0", + type: "turn.failed", + }, + turnContext, ); - expect(flushSpy).toHaveBeenCalledTimes(1); - }); - - it("serializes events per session without blocking other sessions", async () => { - const wildcard = braintrustEveHook({ defineState }).events?.["*"]; - expect(wildcard).toBeDefined(); - - const sessionA: EveHookContext = { session: { id: "session-queue-a" } }; - const sessionB: EveHookContext = { session: { id: "session-queue-b" } }; - const emitA = (event: EveHandleMessageStreamEvent) => - wildcard?.(event, sessionA); - const emitB = (event: EveHandleMessageStreamEvent) => - wildcard?.(event, sessionB); - - await emitA({ - data: { sequence: 0, turnId: "turn-a" }, - type: "turn.started", - }); - await emitB({ - data: { sequence: 0, turnId: "turn-b" }, - type: "turn.started", - }); - - let releaseFirstFlush: (() => void) | undefined; - const firstFlush = new Promise((resolve) => { - releaseFirstFlush = resolve; - }); - const flushSpy = vi - .spyOn(backgroundLogger, "flush") - .mockImplementationOnce(() => firstFlush) - .mockResolvedValue(undefined); - - const doneA = emitA({ - type: "session.completed", - }); - for (let i = 0; i < 10 && flushSpy.mock.calls.length < 1; i++) { - await new Promise((resolve) => setTimeout(resolve, 0)); - } - expect(flushSpy).toHaveBeenCalledTimes(1); - - let queuedEventFinished = false; - const queuedA = Promise.resolve( - emitA({ - data: { - finishReason: "stop", - message: "queued", - sequence: 0, - stepIndex: 0, - turnId: "turn-a", - }, - type: "message.completed", - }), - ).then(() => { - queuedEventFinished = true; - }); - await Promise.resolve(); - expect(queuedEventFinished).toBe(false); - - await emitB({ - data: { sequence: 0, turnId: "turn-b" }, - type: "turn.completed", - }); - expect(queuedEventFinished).toBe(false); - releaseFirstFlush?.(); - await Promise.all([doneA, queuedA]); - expect(queuedEventFinished).toBe(true); + const spans = (await backgroundLogger.drain()) as Array< + Record + >; + expect( + spans.find((span) => span.span_attributes?.name === "eve.step")?.error, + ).toContain("model exploded"); + expect( + spans.find((span) => span.span_attributes?.name === "failing-tool") + ?.error, + ).toContain("tool exploded"); + expect( + spans.find((span) => span.span_attributes?.name === "eve.turn")?.error, + ).toContain("turn exploded"); }); - it("uses deterministic ids and attaches local subagent turns to their tool span", async () => { - const parentEveState = createFakeDefineState(); - const childEveState = createFakeDefineState(); - const parentWildcard = braintrustEveHook({ - defineState: parentEveState.defineState, - }).events?.["*"]; - const childWildcard = braintrustEveHook({ - defineState: childEveState.defineState, - }).events?.["*"]; - expect(parentWildcard).toBeDefined(); - expect(childWildcard).toBeDefined(); - - const parentCtx: EveHookContext = { - session: { id: "session-parent" }, - }; - const childCtx: EveHookContext = { - session: { - id: "session-child", - parent: { - callId: "call-researcher", - sessionId: "session-parent", - turn: { id: "turn-parent" }, - }, - }, - }; - const emitParent = (event: EveHandleMessageStreamEvent) => - parentWildcard?.(event, parentCtx); - const emitChild = (event: EveHandleMessageStreamEvent) => - childWildcard?.(event, childCtx); + it("projects model input without provider-private fields", async () => { + const provider = braintrustEveInstrumentation(); + const turnContext = providerContext(); + const modelContext = providerContext(); - await emitParent({ - data: { sequence: 0, turnId: "turn-parent" }, - type: "turn.started", - }); - await emitParent({ - data: { - message: "Research Eve tracing", + await provider.events["turn.started"]( + { + idempotencyKey: "turn:session-root:turn-0", + rootSessionId: "session-root", sequence: 0, - turnId: "turn-parent", + sessionId: "session-root", + turnId: "turn-0", + type: "turn.started", }, - type: "message.received", - }); - await emitParent({ - data: { sequence: 0, stepIndex: 0, turnId: "turn-parent" }, - type: "step.started", - }); - await emitParent({ - data: { - actions: [ - { - callId: "call-researcher", - input: { message: "Find the relevant section" }, - kind: "subagent-call", - name: "researcher", - subagentName: "researcher", + turnContext, + ); + await provider.events["model.call.started"]( + { + idempotencyKey: "model:attempt-0:0", + input: { + instructions: { + content: "Object system instruction", + providerOptions: { encrypted: "instruction-secret" }, + role: "system", }, - ], - sequence: 0, - stepIndex: 0, - turnId: "turn-parent", - }, - type: "actions.requested", - }); - await emitParent({ - data: { - callId: "call-researcher", - childSessionId: "session-child", - name: "researcher", - sequence: 0, - toolName: "researcher", - turnId: "turn-parent", + messages: [ + { + content: "Hello", + providerOptions: { reasoning_details: "message-secret" }, + role: "user", + }, + { + content: [ + { + providerOptions: { encrypted: "part-secret" }, + text: "Thinking", + type: "reasoning", + }, + { + input: { query: "Eve" }, + providerOptions: { encrypted: "tool-secret" }, + toolCallId: "call-0", + toolName: "search", + type: "tool-call", + }, + { + providerOptions: { encrypted: "custom-secret" }, + type: "custom", + }, + ], + providerOptions: { reasoning_details: "assistant-secret" }, + role: "assistant", + }, + ], + }, + model: { modelId: "gpt-5.4-mini", provider: "openai.responses" }, + scope: ATTEMPT_SCOPE, + type: "model.call.started", }, - type: "subagent.called", - }); + modelContext, + ); - await emitChild({ - data: { sequence: 0, turnId: "turn-child" }, - type: "turn.started", - }); - await emitChild({ - data: { - message: "Find the relevant section", - sequence: 0, - turnId: "turn-child", - }, - type: "message.received", - }); - await emitChild({ - data: { sequence: 0, stepIndex: 0, turnId: "turn-child" }, - type: "step.started", - }); - await emitChild({ - data: { - actions: [ + const spans = (await backgroundLogger.drain()) as Array< + Record + >; + const step = spans.find( + (span) => span.span_attributes?.name === "eve.step", + ); + expect(step?.input).toEqual([ + { content: "Object system instruction", role: "system" }, + { content: "Hello", role: "user" }, + { + content: [ + { text: "Thinking", type: "reasoning" }, { - callId: "call-search", - input: { query: "nested eve" }, - kind: "tool-call", + input: { query: "Eve" }, + toolCallId: "call-0", toolName: "search", + type: "tool-call", }, ], - sequence: 0, - stepIndex: 1, - turnId: "turn-child", - }, - type: "actions.requested", - }); - await emitChild({ - data: { - result: { - callId: "call-search", - kind: "tool-result", - output: { title: "Nested Eve" }, - toolName: "search", - }, - sequence: 0, - status: "completed", - stepIndex: 1, - turnId: "turn-child", - }, - type: "action.result", - }); - await emitChild({ - data: { - finishReason: "stop", - message: "Child found Nested Eve.", - sequence: 0, - stepIndex: 0, - turnId: "turn-child", - }, - type: "message.completed", - }); - await emitChild({ - data: { - finishReason: "stop", - sequence: 0, - stepIndex: 0, - turnId: "turn-child", + role: "assistant", }, - type: "step.completed", - }); - await emitChild({ - data: { sequence: 0, turnId: "turn-child" }, - type: "turn.completed", - }); + ]); + expect(JSON.stringify(step?.input)).not.toContain("providerOptions"); + expect(JSON.stringify(step?.input)).not.toContain("secret"); + }); - await emitParent({ - data: { - callId: "call-researcher", - output: "Child found Nested Eve.", + it("converts inline model and tool-result media to attachments", async () => { + const provider = braintrustEveInstrumentation(); + const actionContext = providerContext(); + await provider.events["turn.started"]( + { + idempotencyKey: "turn:session-root:turn-0", + rootSessionId: "session-root", sequence: 0, - status: "completed", - subagentName: "researcher", - turnId: "turn-parent", + sessionId: "session-root", + turnId: "turn-0", + type: "turn.started", }, - type: "subagent.completed", - }); - await emitParent({ - data: { - result: { - callId: "call-researcher", - kind: "subagent-result", - output: "Child found Nested Eve.", - subagentName: "researcher", + providerContext(), + ); + await provider.events["model.call.started"]( + { + idempotencyKey: "model:attempt-0:0", + input: { + messages: [ + { + content: [ + { + data: "AQID", + filename: "tiny.png", + mediaType: "image/png", + type: "file", + }, + { + image: "DQ4P", + mediaType: "image/png", + type: "image", + }, + { + data: "not valid base64!", + filename: "invalid.png", + mediaType: "image/png", + type: "file", + }, + ], + role: "user", + }, + { + content: [ + { + output: { + type: "content", + value: [ + { + data: "BAUG", + mediaType: "image/png", + type: "image-data", + }, + ], + }, + toolCallId: "call-image", + toolName: "render", + type: "tool-result", + }, + ], + role: "tool", + }, + ], }, - sequence: 0, - status: "completed", - stepIndex: 0, - turnId: "turn-parent", - }, - type: "action.result", - }); - await emitParent({ - data: { sequence: 0, stepIndex: 1, turnId: "turn-parent" }, - type: "step.started", - }); - await emitParent({ - data: { - actions: [ - { - callId: "call-read", - input: { url: "https://eve.dev/docs/guides/instrumentation" }, - kind: "tool-call", - toolName: "read", - }, - ], - sequence: 0, - stepIndex: 1, - turnId: "turn-parent", + model: { modelId: "gpt-5.4-mini", provider: "openai.responses" }, + scope: ATTEMPT_SCOPE, + type: "model.call.started", }, - type: "actions.requested", - }); - await emitParent({ - data: { - result: { - callId: "call-read", - kind: "tool-result", - output: { title: "Runtime context" }, - toolName: "read", + providerContext(), + ); + await provider.events["action.started"]( + { + callId: "call-render", + idempotencyKey: "action:session-root:turn-0:call-render", + input: { + data: "data:image/png;base64,BwgJ", + mediaType: "image/png", + type: "file", }, - sequence: 0, - status: "completed", - stepIndex: 1, - turnId: "turn-parent", - }, - type: "action.result", - }); - await emitParent({ - data: { - finishReason: "tool-calls", - sequence: 0, - stepIndex: 1, - turnId: "turn-parent", - }, - type: "step.completed", - }); - await emitParent({ - data: { sequence: 0, stepIndex: 2, turnId: "turn-parent" }, - type: "step.started", - }); - await emitParent({ - data: { - finishReason: "stop", - message: "Parent used the child result.", - sequence: 0, - stepIndex: 2, - turnId: "turn-parent", + name: "render", + scope: ATTEMPT_SCOPE, + type: "action.started", }, - type: "message.completed", - }); - await emitParent({ - data: { - finishReason: "stop", - sequence: 0, - stepIndex: 2, - turnId: "turn-parent", + actionContext, + ); + await provider.events["action.completed"]( + { + idempotencyKey: "action:session-root:turn-0:call-render", + output: { + output: { + type: "content", + value: [ + { + data: new Uint8Array([10, 11, 12]), + mediaType: "image/png", + type: "image-data", + }, + ], + }, + type: "result", + }, + scope: ATTEMPT_SCOPE, + type: "action.completed", }, - type: "step.completed", - }); - await emitParent({ - data: { sequence: 0, turnId: "turn-parent" }, - type: "turn.completed", - }); + actionContext, + ); const spans = (await backgroundLogger.drain()) as Array< Record >; - const parentTurnId = deterministicEveIdForTest( - "eve:turn", - "session-parent", - "turn-parent", - ); - const childTurnId = deterministicEveIdForTest( - "eve:turn", - "session-child", - "turn-child", - ); - const subagentSpanId = deterministicEveIdForTest( - "eve:subagent", - "session-parent", - "call-researcher", - ); - const parentTurn = spans.find( - (span) => - span.span_attributes?.name === "eve.turn" && - span.span_id === parentTurnId, - ); - const subagentSpans = spans.filter( - (span) => span.span_attributes?.name === "researcher", - ); - const childTurn = spans.find( - (span) => - span.span_attributes?.name === "eve.turn" && - span.span_id === childTurnId, - ); - const childSearch = spans.find( - (span) => - span.span_attributes?.name === "search" && - span.span_parents?.[0] === childTurnId, - ); - const parentRead = spans.find( - (span) => - span.span_attributes?.name === "read" && - span.span_parents?.[0] === parentTurnId, - ); - const parentSteps = spans.filter( - (span) => - span.span_attributes?.name === "eve.step" && - span.span_parents?.[0] === parentTurnId, - ); - - expect(parentTurn).toBeDefined(); - expect(subagentSpans).toHaveLength(1); - expect(subagentSpans[0]?.span_id).toBe(subagentSpanId); - expect(subagentSpans[0]?.input).toEqual({ - message: "Find the relevant section", - }); - expect(childTurn).toBeDefined(); - expect(childSearch).toBeDefined(); - expect(parentRead).toBeDefined(); - expect(parentSteps).toHaveLength(3); - expect(parentTurn?.span_parents).toEqual([]); - expect(parentTurn?.span_id).toBe(parentTurnId); - expect(parentTurn?.span_id).toMatch( - /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/, - ); - expect(parentTurn?.root_span_id).toMatch( - /^([0-9a-f]{32}|[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})$/, - ); - expect(parentTurn?.root_span_id).toBe( - deterministicEveIdForTest("eve:root", "session-parent", "turn-parent"), - ); - expect(subagentSpans[0]?.span_parents).toEqual([parentTurn?.span_id]); - expect(childTurn?.span_parents).toEqual([subagentSpanId]); - expect(childTurn?.root_span_id).toBe(parentTurn?.root_span_id); - expect(parentTurn?.metadata).toEqual({ - "eve.session_id": "session-parent", - }); - expect(subagentSpans[0]?.metadata).toEqual({ - "eve.session_id": "session-parent", - }); - expect(childTurn?.metadata).toEqual({ - "eve.session_id": "session-child", - }); - expect(childSearch?.metadata).toEqual({ - "eve.session_id": "session-child", - }); - expect(parentRead?.metadata).toEqual({ - "eve.session_id": "session-parent", - }); - expect(childSearch?.span_parents).toEqual([childTurn?.span_id]); - expect(childSearch?.span_id).toBe( - deterministicEveIdForTest( - "eve:tool", - "session-child", - "turn-child", - "call-search", - ), + const step = spans.find( + (span) => span.span_attributes?.name === "eve.step", ); - expect(parentRead?.span_id).toBe( - deterministicEveIdForTest( - "eve:tool", - "session-parent", - "turn-parent", - "call-read", - ), + const inputAttachment = step?.input?.[0]?.content?.[0]?.data; + const imageAttachment = step?.input?.[0]?.content?.[1]?.image; + const unconvertedFile = step?.input?.[0]?.content?.[2]?.data; + const outputAttachment = + step?.input?.[1]?.content?.[0]?.output?.value?.[0]?.data; + const action = spans.find( + (span) => span.span_attributes?.name === "render", ); - expect(parentSteps.map((span) => span.span_id)).toEqual([ - deterministicEveIdForTest( - "eve:step", - "session-parent", - "turn-parent", - "0", - ), - deterministicEveIdForTest( - "eve:step", - "session-parent", - "turn-parent", - "1", - ), - deterministicEveIdForTest( - "eve:step", - "session-parent", - "turn-parent", - "2", - ), - ]); - expect(spans.map((span) => span.span_attributes?.name)).toEqual([ - "eve.turn", - "eve.step", - "researcher", - "eve.turn", - "eve.step", - "search", - "eve.step", - "read", - "eve.step", - ]); + const actionInputAttachment = action?.input?.data; + const actionOutputAttachment = action?.output?.value?.[0]?.data; + expect(inputAttachment).toBeInstanceOf(Attachment); + expect(inputAttachment.reference).toMatchObject({ + content_type: "image/png", + filename: "tiny.png", + type: "braintrust_attachment", + }); + expect(imageAttachment).toBeInstanceOf(Attachment); + expect(imageAttachment.reference).toMatchObject({ + content_type: "image/png", + type: "braintrust_attachment", + }); + expect(unconvertedFile).toBe("not valid base64!"); + expect(outputAttachment).toBeInstanceOf(Attachment); + expect(outputAttachment.reference).toMatchObject({ + content_type: "image/png", + filename: "attachment.png", + type: "braintrust_attachment", + }); + expect(actionInputAttachment).toBeInstanceOf(Attachment); + expect(actionInputAttachment.reference).toMatchObject({ + content_type: "image/png", + type: "braintrust_attachment", + }); + expect(actionOutputAttachment).toBeInstanceOf(Attachment); + expect(actionOutputAttachment.reference).toMatchObject({ + content_type: "image/png", + type: "braintrust_attachment", + }); + expect(JSON.stringify(step?.input)).not.toContain("AQID"); + expect(JSON.stringify(step?.input)).not.toContain("BAUG"); + }); - backgroundLogger = _exportsForTestingOnly.useTestBackgroundLogger(); - initLogger({ - projectName: "eve-plugin.test.ts", - projectId: "test-project-id", - }); - const replay = braintrustEveHook({ - defineState: parentEveState.defineState, - }).events?.["*"]; - await replay?.( + it("preserves arbitrary action outputs with type fields", async () => { + const provider = braintrustEveInstrumentation(); + const actionContext = providerContext(); + await provider.events["turn.started"]( { - data: { sequence: 0, turnId: "turn-parent" }, + idempotencyKey: "turn:session-root:turn-0", + rootSessionId: "session-root", + sequence: 0, + sessionId: "session-root", + turnId: "turn-0", type: "turn.started", }, - parentCtx, + providerContext(), ); - await replay?.( + await provider.events["action.started"]( { - data: { sequence: 0, turnId: "turn-parent" }, - type: "turn.completed", + callId: "call-text", + idempotencyKey: "action:session-root:turn-0:call-text", + name: "text-result", + scope: ATTEMPT_SCOPE, + type: "action.started", + }, + actionContext, + ); + await provider.events["action.completed"]( + { + idempotencyKey: "action:session-root:turn-0:call-text", + output: { + output: { text: "preserve me", type: "text" }, + type: "result", + }, + scope: ATTEMPT_SCOPE, + type: "action.completed", }, - parentCtx, + actionContext, ); - const replaySpans = (await backgroundLogger.drain()) as Array< + + const spans = (await backgroundLogger.drain()) as Array< Record >; expect( - replaySpans.find((span) => span.span_id === parentTurn?.span_id), - ).toMatchObject({ _is_merge: true }); + spans.find((span) => span.span_attributes?.name === "text-result") + ?.output, + ).toEqual({ text: "preserve me", type: "text" }); }); - it("does not parent an Eve turn under the active Braintrust span", async () => { - const wildcard = braintrustEveHook({ defineState }).events?.["*"]; - expect(wildcard).toBeDefined(); + it.each([ + ["step.attempt.completed", undefined], + ["step.attempt.failed", new Error("attempt exploded")], + ] as const)("closes an open model span on %s", async (type, error) => { + const provider = braintrustEveInstrumentation(); + await provider.events["turn.started"]( + { + idempotencyKey: "turn:session-root:turn-0", + rootSessionId: "session-root", + sequence: 0, + sessionId: "session-root", + turnId: "turn-0", + type: "turn.started", + }, + providerContext(), + ); + await provider.events["model.call.started"]( + { + idempotencyKey: "model:attempt-0:0", + model: { modelId: "gpt-5.4-mini", provider: "openai.responses" }, + scope: ATTEMPT_SCOPE, + type: "model.call.started", + }, + providerContext(), + ); - const ctx: EveHookContext = { - session: { id: "session-wrapped" }, - }; - const parent = startSpan({ name: "workflow" }); - await withCurrent(parent, async () => { - await wildcard?.( + if (type === "step.attempt.completed") { + await provider.events[type]( { - data: { sequence: 0, turnId: "turn-wrapped" }, - type: "turn.started", + idempotencyKey: "step:attempt-0", + scope: ATTEMPT_SCOPE, + type, }, - ctx, + providerContext(), ); - await wildcard?.( + } else { + await provider.events[type]( { - data: { sequence: 0, turnId: "turn-wrapped" }, - type: "turn.completed", + error, + idempotencyKey: "step:attempt-0", + scope: ATTEMPT_SCOPE, + type, }, - ctx, + providerContext(), ); - }); - parent.end(); + } const spans = (await backgroundLogger.drain()) as Array< Record >; - const turn = spans.find( - (span) => span.span_attributes?.name === "eve.turn", - ); - expect(turn?.span_id).toBe( - deterministicEveIdForTest("eve:turn", "session-wrapped", "turn-wrapped"), - ); - expect(turn?.span_parents).toEqual([]); - expect(turn?.root_span_id).toBe( - deterministicEveIdForTest("eve:root", "session-wrapped", "turn-wrapped"), + const step = spans.find( + (span) => span.span_attributes?.name === "eve.step", ); - expect(turn?.root_span_id).not.toBe(parent.rootSpanId); - }); - - it("does not throw when Eve emits malformed events or failures", async () => { - const wildcard = braintrustEveHook({ defineState }).events?.["*"]; - expect(wildcard).toBeDefined(); - - await expect( - wildcard?.({ bad: true } as never, {} as never), - ).resolves.toBeUndefined(); - await expect( - wildcard?.( - { - data: { - code: "boom", - message: "step failed", - sequence: 0, - stepIndex: 0, - turnId: "turn-missing-session", - }, - type: "step.failed", - }, - {} as never, - ), - ).resolves.toBeUndefined(); - await expect( - wildcard?.( - { - data: { runtime: { modelId: 123 } }, - type: "session.started", - } as never, - { session: { id: "session-malformed-runtime" } }, - ), - ).resolves.toBeUndefined(); - - const spans = (await backgroundLogger.drain()) as Array< - Record - >; - expect(spans).toEqual([]); + expect(step?.metrics?.end).toEqual(expect.any(Number)); + if (error !== undefined) { + expect(step?.error).toContain("attempt exploded"); + } }); }); diff --git a/js/src/instrumentation/plugins/eve-plugin.ts b/js/src/instrumentation/plugins/eve-plugin.ts index ef18f8913..8729c6a48 100644 --- a/js/src/instrumentation/plugins/eve-plugin.ts +++ b/js/src/instrumentation/plugins/eve-plugin.ts @@ -1,7 +1,9 @@ import { debugLogger } from "../../debug-logger"; import { + Attachment, NOOP_SPAN, _internalStartSpanWithInitialMerge, + currentLogger, flush, updateSpan, withCurrent, @@ -14,161 +16,105 @@ import { } from "../../span-origin"; import { SpanTypeAttribute, isObject } from "../../../util/index"; import { getCurrentUnixTimestamp } from "../../util"; +import { + convertDataToBlob, + getExtensionFromMediaType, + processInputAttachments, +} from "../../wrappers/attachment-utils"; import type { - EveAssistantStepFinishReason, - EveActionResultError, - EveHandleMessageStreamEvent, - EveHookContext, - EveHookDefinition, + EveInstrumentationActionCompletedEvent, + EveInstrumentationActionFailedEvent, + EveInstrumentationActionStartedEvent, + EveInstrumentationContentPart, EveInstrumentationDefinition, + EveInstrumentationHandlerContext, + EveInstrumentationModelCallCompletedEvent, + EveInstrumentationModelCallFailedEvent, + EveInstrumentationModelCallStartedEvent, EveInstrumentationModelInput, - EveInstrumentationStepStartedEventInput, - EveModelMessage, - EveModelMessageContentPart, - EveRuntimeActionRequest, - EveRuntimeActionResult, - EveRuntimeToolCallActionRequest, - EveRuntimeToolResultActionResult, + EveInstrumentationStepAttemptCompletedEvent, + EveInstrumentationStepAttemptFailedEvent, + EveInstrumentationTurnFailedEvent, + EveInstrumentationTurnSettledEvent, + EveInstrumentationTurnStartedEvent, + EveInstrumentationUsage, } from "../../vendor-sdk-types/eve"; -type SpanState = { - metadata: Record; - span: EveSpan; -}; - type EveSpan = Pick; -type EveSpanStartEvent = { - readonly created: string; - readonly metrics: { readonly start: number }; - readonly span_attributes: Record; - readonly span_parents: string[]; -}; - type EveSpanReference = { - readonly endTime?: number; readonly exported: string; readonly rootSpanId: string; - readonly rowId: string; readonly spanId: string; - readonly startEvent?: EveSpanStartEvent; -}; - -type StepState = SpanState & { - input?: unknown; - metrics: Record; - output?: unknown; - reasoning: readonly EveReasoningBlock[]; + readonly turnMetricContributions?: Record>; }; -type TurnState = SpanState & { +type TurnState = { key: string; - metrics: Record; - output?: unknown; - sessionId: string; - stepsByIndex: Map; - turnId: string; -}; - -type ToolState = SpanState & { - endedByTurn?: boolean; - turnKey: string; -}; - -type EveEntityKind = "step" | "subagent" | "tool" | "turn"; - -type EveStateHandle = { - get(): T; - update(fn: (current: T) => T): void; -}; - -type EveDefineState = (name: string, initial: () => T) => EveStateHandle; - -type EveTraceState = { metadata: Record; - reasoningBlocks: readonly (EveReasoningBlock & { key: string })[]; - spanReferences: readonly EveSpanReference[]; - stepStarts: readonly { - ordinal: number; - open: boolean; - stepIndex: number; - turnId: string; - }[]; - llmInputs: readonly { - input: CapturedEveModelInput; - key: string; - }[]; + rootSpanId: string; + rowId: string; + span: EveSpan; + spanId: string; }; -type EveReasoningBlock = { - content: string; - eventAt?: string; -}; +type EveInstrumentationEvent = + | EveInstrumentationActionCompletedEvent + | EveInstrumentationActionFailedEvent + | EveInstrumentationActionStartedEvent + | EveInstrumentationModelCallCompletedEvent + | EveInstrumentationModelCallFailedEvent + | EveInstrumentationModelCallStartedEvent + | EveInstrumentationStepAttemptCompletedEvent + | EveInstrumentationStepAttemptFailedEvent + | EveInstrumentationTurnFailedEvent + | EveInstrumentationTurnSettledEvent + | EveInstrumentationTurnStartedEvent; -const EVE_TRACE_STATE_KEY = "braintrust.eve.tracing"; const MAX_EVE_CACHE_ENTRIES = 10_000; -const MAX_STORED_LLM_INPUTS = 100; -const MAX_STORED_REASONING_BLOCKS = 100; -const MAX_STORED_SPAN_REFERENCES = 10_000; -const MAX_STORED_STEP_STARTS = 10_000; - -type CapturedEveModelMessage = { - content: string | readonly Record[]; - role: EveModelMessage["role"]; -}; - -type CapturedEveModelInput = readonly CapturedEveModelMessage[]; - -/** Manual hook instrumentation for eve runtime stream events. */ -export function braintrustEveHook(options: { - defineState: EveDefineState; - metadata?: Record; -}): EveHookDefinition { - const state = options.defineState(EVE_TRACE_STATE_KEY, emptyEveTraceState); - const bridge = new EveBridge(state); - return { - events: { - "*": async (event: EveHandleMessageStreamEvent, ctx: EveHookContext) => { - await bridge.handle(event, ctx, options.metadata); - }, - }, - }; -} - -/** Eve instrumentation helper for logger setup and durable LLM input capture. */ -export function braintrustEveInstrumentation(options: { - defineState: EveDefineState; - setup?: EveInstrumentationDefinition["setup"]; -}): EveInstrumentationDefinition { - const state = options.defineState(EVE_TRACE_STATE_KEY, emptyEveTraceState); +const eveRootSpanIdsByTurnKey = new LRUCache({ + max: MAX_EVE_CACHE_ENTRIES, +}); + +/** + * Creates an Eve instrumentation provider. + * + * Eve 0.34.0 introduced the provider event API used here. Older Eve hook and + * single-file instrumentation layouts are intentionally unsupported. + */ +export function braintrustEveInstrumentation( + options: { + metadata?: Record; + setup?: EveInstrumentationDefinition["setup"]; + } = {}, +): EveInstrumentationDefinition { + const bridge = new EveBridge(options.metadata); return { + capture: "content", events: { - "step.started": (input: EveInstrumentationStepStartedEventInput) => { - try { - captureEveModelInput(state, input); - } catch (error) { - debugLogger.warn("Error in Eve LLM input capture:", error); - } - }, + "action.completed": (event, context) => bridge.handle(event, context), + "action.failed": (event, context) => bridge.handle(event, context), + "action.started": (event, context) => bridge.handle(event, context), + "model.call.completed": (event, context) => bridge.handle(event, context), + "model.call.failed": (event, context) => bridge.handle(event, context), + "model.call.started": (event, context) => bridge.handle(event, context), + "step.attempt.completed": (event, context) => + bridge.handle(event, context), + "step.attempt.failed": (event, context) => bridge.handle(event, context), + "turn.cancelled": (event, context) => bridge.handle(event, context), + "turn.completed": (event, context) => bridge.handle(event, context), + "turn.failed": (event, context) => bridge.handle(event, context), + "turn.started": (event, context) => bridge.handle(event, context), }, - recordInputs: false, - recordOutputs: false, + flush: () => flush(), setup: options.setup, }; } -function isEveHandleMessageStreamEvent( - event: unknown, -): event is EveHandleMessageStreamEvent { - return isObject(event) && typeof event["type"] === "string"; -} - class ResumedEveSpan implements EveSpan { private endTime: number | undefined; - constructor(private readonly reference: EveSpanReference) { - this.endTime = reference.endTime; - } + constructor(private readonly reference: EveSpanReference) {} get rootSpanId(): string { return this.reference.rootSpanId; @@ -179,16 +125,9 @@ class ResumedEveSpan implements EveSpan { } log(event: Parameters[0]): void { - const metrics = { - ...this.reference.startEvent?.metrics, - ...(this.endTime === undefined ? {} : { end: this.endTime }), - ...event.metrics, - }; updateSpan({ exported: this.reference.exported, - ...this.reference.startEvent, ...event, - ...(Object.keys(metrics).length > 0 ? { metrics } : {}), }); } @@ -202,2109 +141,1061 @@ class ResumedEveSpan implements EveSpan { } class EveBridge { - constructor(private readonly state: EveStateHandle) {} - - private eventQueuesBySession = new Map>(); - private completedToolKeys = new LRUCache({ - max: MAX_EVE_CACHE_ENTRIES, - }); - private toolsByCallKey = new LRUCache({ - max: MAX_EVE_CACHE_ENTRIES, - }); private turnsByKey = new LRUCache({ max: MAX_EVE_CACHE_ENTRIES, }); + private turnsStartingByKey = new Map>(); + private openModelsByAttempt = new Map>(); + private turnMetricContributionsByKey = new Map< + string, + Record> + >(); - private async startEveSpan( - args: Parameters[0], - ): Promise { - const rowId = args?.event?.id; - const reference = - typeof rowId === "string" && - readEveTraceState(this.state).spanReferences.find( - (candidate) => candidate.rowId === rowId, - ); - if (reference) { - return new ResumedEveSpan(reference); - } - - const startTime = args?.startTime ?? getCurrentUnixTimestamp(); - const parentSpanIds = args?.parentSpanIds; - const startEvent: EveSpanStartEvent = { - created: new Date().toISOString(), - metrics: { start: startTime }, - span_attributes: { - ...(args?.name ? { name: args.name } : {}), - ...(args?.type ? { type: args.type } : {}), - ...args?.spanAttributes, - }, - span_parents: parentSpanIds - ? "spanId" in parentSpanIds - ? [parentSpanIds.spanId] - : parentSpanIds.parentSpanIds - : [], - }; - const span = withCurrent(NOOP_SPAN, () => - _internalStartSpanWithInitialMerge( - withSpanInstrumentationName( - { ...args, startTime }, - INSTRUMENTATION_NAMES.EVE, - ), - ), - ); - if (typeof rowId !== "string") { - return span; - } + constructor(private readonly configuredMetadata?: Record) {} + async handle( + event: EveInstrumentationEvent, + context: EveInstrumentationHandlerContext, + ): Promise { try { - const exported = await span.export(); - const reference = { - exported, - rootSpanId: span.rootSpanId, - rowId, - spanId: span.spanId, - startEvent, - }; - this.state.update((current) => { - const normalized = normalizeEveTraceState(current); - return normalized.spanReferences.some( - (candidate) => candidate.rowId === rowId, - ) - ? normalized - : { - ...normalized, - spanReferences: [...normalized.spanReferences, reference].slice( - -MAX_STORED_SPAN_REFERENCES, - ), - }; - }); + switch (event.type) { + case "turn.started": + await this.getOrStartTurn(event, context); + return; + case "model.call.started": + await this.handleModelCallStarted(event, context); + return; + case "model.call.completed": + await this.handleModelCallCompleted(event, context); + return; + case "model.call.failed": + await this.handleModelCallFailed(event, context); + return; + case "step.attempt.completed": + case "step.attempt.failed": + this.handleStepAttemptTerminal(event); + return; + case "action.started": + await this.handleActionStarted(event, context); + return; + case "action.completed": + await this.handleActionCompleted(event, context); + return; + case "action.failed": + await this.handleActionFailed(event, context); + return; + case "turn.cancelled": + case "turn.completed": + await this.handleTurnSettled(event, context); + return; + case "turn.failed": + await this.handleTurnFailed(event, context); + return; + } } catch (error) { - debugLogger.warn("Error exporting Eve span for resumption:", error); + debugLogger.warn("Error in Eve instrumentation provider:", error); } - return span; } - private async startEveChildSpan( - parent: EveSpan, - args: Parameters[0], - ): Promise { - return await this.startEveSpan({ - ...args, + private async handleModelCallStarted( + event: EveInstrumentationModelCallStartedEvent, + context: EveInstrumentationHandlerContext, + ): Promise { + const turn = await this.turnForScope(event.scope, context); + const metadata = { + ...this.metadata(event.scope.sessionId), + ...modelMetadata(event.model.modelId, event.model.provider), + }; + const input = capturedModelInput(event.input); + const ids = await generateEveIds("step", event.idempotencyKey); + const span = await this.startEveSpan(context, { + event: { + id: ids.rowId, + ...(input === undefined ? {} : { input }), + metadata, + }, + name: "eve.step", parentSpanIds: { - rootSpanId: parent.rootSpanId, - spanId: parent.spanId, + rootSpanId: turn.rootSpanId, + spanId: turn.spanId, }, + spanAttributes: { type: SpanTypeAttribute.LLM }, + spanId: ids.spanId, }); - } + const reference = readSpanReference(context.state.get()); + if (reference !== undefined) { + context.state.set({ + ...reference, + turnMetricContributions: + this.turnMetricContributionsByKey.get(turn.key) ?? {}, + }); + } + const openModels = + this.openModelsByAttempt.get(event.scope.attemptId) ?? new Map(); + openModels.set(event.idempotencyKey, span); + this.openModelsByAttempt.set(event.scope.attemptId, openModels); - private stepOrdinal( - event: Extract, - ): number { - let ordinal = 0; - this.state.update((current) => { - const state = normalizeEveTraceState(current); - const previous = state.stepStarts - .filter( - (entry) => - entry.turnId === event.data.turnId && - entry.stepIndex === event.data.stepIndex, - ) - .at(-1); - if (previous?.open) { - ordinal = previous.ordinal; - return state; + if (input !== undefined) { + const initialInput = initialTurnInput(input); + if (initialInput !== undefined) { + this.updateTurn(turn, { input: initialInput }); } - - ordinal = state.stepStarts.filter( - (entry) => entry.turnId === event.data.turnId, - ).length; - return { - ...state, - stepStarts: [ - ...state.stepStarts, - { - open: true, - ordinal, - stepIndex: event.data.stepIndex, - turnId: event.data.turnId, - }, - ].slice(-MAX_STORED_STEP_STARTS), - }; - }); - return ordinal; + } } - private markStepEnded(turnId: string, stepIndex: number): void { - this.state.update((current) => { - const state = normalizeEveTraceState(current); - let index = -1; - for (let i = state.stepStarts.length - 1; i >= 0; i--) { - const entry = state.stepStarts[i]; - if (entry?.turnId === turnId && entry.stepIndex === stepIndex) { - index = i; - break; - } - } - if (index < 0 || !state.stepStarts[index]?.open) { - return state; - } - return { - ...state, - stepStarts: state.stepStarts.map((entry, entryIndex) => - entryIndex === index ? { ...entry, open: false } : entry, - ), - }; - }); + private async handleModelCallCompleted( + event: EveInstrumentationModelCallCompletedEvent, + context: EveInstrumentationHandlerContext, + ): Promise { + const turn = await this.turnForScope(event.scope, context); + this.restoreTurnMetricContributions(turn.key, context.state.get()); + const span = await this.resumeModelSpan(event, context, turn); + const metrics = usageMetrics(event.usage); + const output = modelOutput(event.finishReason, event.content); + span.log({ metrics, output }); + span.end(); + this.closeOpenModel(event.scope.attemptId, event.idempotencyKey); + this.updateTurnMetrics(turn, event.idempotencyKey, metrics); + + const finalOutput = finalText(event.content); + if ( + finalOutput !== undefined && + normalizedFinishReason(event.finishReason) !== "tool_calls" + ) { + this.updateTurn(turn, { output: finalOutput }); + } } - async handle( - event: unknown, - ctx: unknown, - hookMetadata?: Record, + private async handleModelCallFailed( + event: EveInstrumentationModelCallFailedEvent, + context: EveInstrumentationHandlerContext, ): Promise { - if (!isEveHandleMessageStreamEvent(event)) { - return; - } - const run = async () => { - try { - if (!(await this.handleEvent(event, ctx, hookMetadata))) { - return; - } - if (event.type === "session.failed") { - const sessionId = event.data.sessionId || sessionIdFromContext(ctx); - await this.flushInstrumentation(); - if (sessionId) { - this.cleanupSession(sessionId); - } - } else if (event.type === "session.completed") { - const sessionId = sessionIdFromContext(ctx); - await this.flushInstrumentation(); - if (sessionId) { - this.cleanupSession(sessionId); - } - } - } catch (error) { - debugLogger.warn("Error in Eve hook instrumentation:", error); - } - }; + const turn = await this.turnForScope(event.scope, context); + const span = await this.resumeModelSpan(event, context, turn); + span.log({ error: toError(event.error, "Eve model call failed") }); + span.end(); + this.closeOpenModel(event.scope.attemptId, event.idempotencyKey); + } - const sessionId = - event.type === "session.failed" - ? event.data.sessionId || sessionIdFromContext(ctx) - : sessionIdFromContext(ctx); - if (!sessionId) { - await run(); + private handleStepAttemptTerminal( + event: + | EveInstrumentationStepAttemptCompletedEvent + | EveInstrumentationStepAttemptFailedEvent, + ): void { + const openModels = this.openModelsByAttempt.get(event.scope.attemptId); + if (openModels === undefined) { return; } - - const previous = this.eventQueuesBySession.get(sessionId); - const queued = previous ? previous.then(run) : run(); - this.eventQueuesBySession.set(sessionId, queued); - try { - await queued; - } finally { - if (this.eventQueuesBySession.get(sessionId) === queued) { - this.eventQueuesBySession.delete(sessionId); + for (const span of openModels.values()) { + if (event.type === "step.attempt.failed") { + span.log({ + error: toError(event.error, "Eve step attempt failed"), + }); } + span.end(); } + this.openModelsByAttempt.delete(event.scope.attemptId); } - private async handleEvent( - event: EveHandleMessageStreamEvent, - ctx: unknown, - hookMetadata?: Record, - ): Promise { - switch (event.type) { - case "session.started": - this.handleSessionStarted(event, ctx, hookMetadata); - return true; - case "turn.started": - await this.handleTurnStarted(event, ctx, hookMetadata); - return true; - case "message.received": - await this.handleMessageReceived(event, ctx, hookMetadata); - return true; - case "step.started": - await this.handleStepStarted(event, ctx, hookMetadata); - return true; - case "reasoning.completed": - this.handleReasoningCompleted(event, ctx); - return true; - case "message.completed": - this.handleMessageCompleted(event, ctx); - return true; - case "result.completed": - this.handleResultCompleted(event, ctx); - return true; - case "actions.requested": - await this.handleActionsRequested(event, ctx, hookMetadata); - return true; - case "action.result": - await this.handleActionResult(event, ctx, hookMetadata); - return true; - case "subagent.called": - await this.handleSubagentCalled(event, ctx, hookMetadata); - return true; - case "subagent.completed": - await this.handleSubagentCompleted(event, ctx, hookMetadata); - return true; - case "step.completed": - this.handleStepCompleted(event, ctx); - return true; - case "step.failed": - this.handleStepFailed(event, ctx); - return true; - case "turn.completed": - this.handleTurnCompleted(event, ctx); - return true; - case "turn.failed": - this.handleTurnFailed(event, ctx); - return true; - case "session.failed": - this.handleSessionFailed(event, ctx); - return true; - case "session.completed": - this.handleSessionCompleted(event, ctx); - return true; - default: - return false; - } + private async handleActionStarted( + event: EveInstrumentationActionStartedEvent, + context: EveInstrumentationHandlerContext, + ): Promise { + const turn = await this.turnForScope(event.scope, context); + const metadata = turn.metadata; + const input = processInputAttachments(event.input); + const ids = await generateEveIds( + "action", + event.scope.sessionId, + event.scope.turnId, + event.callId, + ); + await this.startEveSpan(context, { + event: { + id: ids.rowId, + ...(input === undefined ? {} : { input }), + metadata, + }, + name: event.name, + parentSpanIds: { + rootSpanId: turn.rootSpanId, + spanId: turn.spanId, + }, + spanAttributes: { type: SpanTypeAttribute.TOOL }, + spanId: ids.spanId, + }); } - private handleSessionStarted( - event: Extract, - ctx: unknown, - hookMetadata?: Record, - ): void { - const sessionId = sessionIdFromContext(ctx); - if (!sessionId) { - return; + private async handleActionCompleted( + event: EveInstrumentationActionCompletedEvent, + context: EveInstrumentationHandlerContext, + ): Promise { + const span = await this.resumeActionSpan(event, context); + if (event.output.type === "error") { + span.log({ + error: toError(event.output.error, "Eve action failed"), + }); + } else if (event.output.output !== undefined) { + span.log({ output: processInputAttachments(event.output.output) }); } + span.end(actionEndArgs(event.acceptedAtMs)); + } - const metadata = { - ...(hookMetadata ?? {}), - ...modelMetadataFromRuntime(event.data.runtime), - }; - this.state.update((current) => { - const normalized = normalizeEveTraceState(current); - return { - ...normalized, - metadata: { ...normalized.metadata, ...metadata }, - }; + private async handleActionFailed( + event: EveInstrumentationActionFailedEvent, + context: EveInstrumentationHandlerContext, + ): Promise { + const span = await this.resumeActionSpan(event, context); + span.log({ + error: toError( + event.error, + event.errorCode === undefined + ? "Eve action " + event.outcome + : event.errorCode + ": Eve action " + event.outcome, + ), }); - for (const [key, turn] of this.turnsByKey) { - if (!key.startsWith(`${sessionId}:`)) { - continue; - } - - turn.metadata = { ...turn.metadata, ...metadata }; - turn.span.log({ metadata: turn.metadata }); - for (const step of turn.stepsByIndex.values()) { - step.metadata = { ...step.metadata, ...metadata }; - step.span.log({ metadata: step.metadata }); - } - } + span.end(actionEndArgs(event.acceptedAtMs)); } - private async handleTurnStarted( - event: Extract, - ctx: unknown, - hookMetadata?: Record, + private async handleTurnSettled( + event: EveInstrumentationTurnSettledEvent, + context: EveInstrumentationHandlerContext, ): Promise { - const sessionId = sessionIdFromContext(ctx); - if (!sessionId) { - return; - } + const turn = await this.resumeTurn(event, context); + turn.span.end(); + this.turnsByKey.delete(turn.key); + this.turnMetricContributionsByKey.delete(turn.key); + } - const key = turnKey(sessionId, event.data.turnId); - const metadata = { - ...readEveTraceState(this.state).metadata, - ...(hookMetadata ?? {}), - "eve.session_id": sessionId, - }; - const existing = this.turnsByKey.get(key); - if (existing) { - existing.metadata = { ...existing.metadata, ...metadata }; - existing.span.log({ metadata: existing.metadata }); - return; - } + private async handleTurnFailed( + event: EveInstrumentationTurnFailedEvent, + context: EveInstrumentationHandlerContext, + ): Promise { + const turn = await this.resumeTurn(event, context); + turn.span.log({ + error: toError(event.error, "Eve turn failed"), + }); + turn.span.end(); + this.turnsByKey.delete(turn.key); + this.turnMetricContributionsByKey.delete(turn.key); + } - const span = await this.startTurnSpan(sessionId, event, ctx, metadata); - span.log({ metadata }); - this.turnsByKey.set(key, { - key, - metadata, - metrics: {}, - sessionId, - span, - stepsByIndex: new Map(), - turnId: event.data.turnId, + private async startTurnSpan( + event: EveInstrumentationTurnStartedEvent, + context: EveInstrumentationHandlerContext, + metadata: Record, + ): Promise { + const ids = await generateEveIds("turn", event.sessionId, event.turnId); + const parentSpanId = + event.parentLineage === undefined + ? undefined + : await deterministicEveId( + "eve:action", + event.parentLineage.sessionId, + event.parentLineage.turnId, + event.parentLineage.callId, + ); + const parentTurn = + event.parentLineage === undefined + ? undefined + : this.turnsByKey.get( + turnKey(event.parentLineage.sessionId, event.parentLineage.turnId), + ); + const parentRootSpanId = + event.parentLineage === undefined + ? undefined + : eveRootSpanIdsByTurnKey.get( + turnKey(event.parentLineage.sessionId, event.parentLineage.turnId), + ); + const rootSpanId = + event.parentLineage === undefined + ? await deterministicEveId("eve:root", event.sessionId, event.turnId) + : (parentTurn?.rootSpanId ?? + parentRootSpanId ?? + (typeof event.parentTraceContext?.traceId === "string" && + event.parentTraceContext.traceId.length > 0 + ? event.parentTraceContext.traceId + : await deterministicEveId( + "eve:root", + event.parentLineage.sessionId, + event.parentLineage.turnId, + ))); + return await this.startEveSpan(context, { + event: { id: ids.rowId, metadata }, + name: "eve.turn", + parentSpanIds: + parentSpanId === undefined + ? { parentSpanIds: [], rootSpanId } + : { rootSpanId, spanId: parentSpanId }, + spanAttributes: { type: SpanTypeAttribute.TASK }, + spanId: ids.spanId, }); } - private async handleMessageReceived( - event: Extract, - ctx: unknown, - hookMetadata?: Record, - ): Promise { - const turn = await this.ensureTurn(event, ctx, hookMetadata); - if (!turn) { - return; + private async startEveSpan( + context: EveInstrumentationHandlerContext, + args: Parameters[0], + ): Promise { + const reference = readSpanReference(context.state.get()); + if (reference !== undefined) { + return new ResumedEveSpan(reference); } - const input = [{ content: event.data.message, role: "user" }]; - turn.span.log({ input }); + const startTime = args?.startTime ?? getCurrentUnixTimestamp(); + const span = withCurrent(NOOP_SPAN, () => + _internalStartSpanWithInitialMerge( + withSpanInstrumentationName( + { ...args, startTime }, + INSTRUMENTATION_NAMES.EVE, + ), + ), + ); + + try { + const reference: EveSpanReference = { + exported: await span.export(), + rootSpanId: span.rootSpanId, + spanId: span.spanId, + }; + context.state.set(reference); + } catch (error) { + debugLogger.warn("Error exporting Eve span for resumption:", error); + } + return span; } - private async handleStepStarted( - event: Extract, - ctx: unknown, - hookMetadata?: Record, - ): Promise { - const turn = await this.ensureTurn(event, ctx, hookMetadata); - const sessionId = sessionIdFromContext(ctx); - if (!turn || !sessionId) { - return; + private async getOrStartTurn( + event: EveInstrumentationTurnStartedEvent, + context: EveInstrumentationHandlerContext, + ): Promise { + const key = turnKey(event.sessionId, event.turnId); + const existing = this.turnsByKey.get(key); + if (existing !== undefined) { + return existing; } - const existing = turn.stepsByIndex.get(event.data.stepIndex); - if (existing) { - existing.span.log({ - ...(existing.input !== undefined ? { input: existing.input } : {}), - metadata: existing.metadata, - metrics: existing.metrics, - output: existing.output, - }); - const endTime = eventTime(event); - existing.span.end(endTime === undefined ? undefined : { endTime }); - this.markStepEnded(event.data.turnId, event.data.stepIndex); - clearStoredEveReasoning( - this.state, - sessionId, - event.data.turnId, - event.data.stepIndex, - ); + const starting = this.turnsStartingByKey.get(key); + if (starting !== undefined) { + return await starting; } - const stepOrdinal = this.stepOrdinal(event); - const metadata = { ...turn.metadata }; - const input = consumeCapturedEveModelInput( - this.state, - sessionId, - event.data.turnId, - event.data.stepIndex, - ); - const reasoning = readStoredEveReasoning( - this.state, - sessionId, - event.data.turnId, - event.data.stepIndex, - ); - const output = mergeEveReasoning(undefined, reasoning); - const { rowId: eventId, spanId } = await generateEveIds( - "step", - sessionId, - event.data.turnId, - String(stepOrdinal), - ); - const span = await this.startEveChildSpan(turn.span, { - event: { - id: eventId, - ...(input !== undefined ? { input } : {}), + const metadata = this.metadata(event.sessionId); + const start = (async () => { + const ids = await generateEveIds("turn", event.sessionId, event.turnId); + const span = await this.startTurnSpan(event, context, metadata); + const turn = { + key, metadata, + rootSpanId: span.rootSpanId, + rowId: ids.rowId, + span, + spanId: span.spanId, + }; + this.turnsByKey.set(key, turn); + eveRootSpanIdsByTurnKey.set(key, turn.rootSpanId); + return turn; + })(); + this.turnsStartingByKey.set(key, start); + try { + return await start; + } finally { + if (this.turnsStartingByKey.get(key) === start) { + this.turnsStartingByKey.delete(key); + } + } + } + + private async resumeTurn( + event: + | EveInstrumentationTurnSettledEvent + | EveInstrumentationTurnFailedEvent, + context: EveInstrumentationHandlerContext, + ): Promise { + const key = turnKey(event.sessionId, event.turnId); + const existing = this.turnsByKey.get(key); + if (existing !== undefined) { + return existing; + } + const starting = this.turnsStartingByKey.get(key); + if (starting !== undefined) { + return await starting; + } + + const metadata = this.metadata(event.sessionId); + const span = await this.startTurnSpan( + { + idempotencyKey: event.idempotencyKey, + rootSessionId: event.sessionId, + sequence: 0, + sessionId: event.sessionId, + turnId: event.turnId, + type: "turn.started", }, - name: "eve.step", - spanAttributes: { type: SpanTypeAttribute.LLM }, - spanId, - startTime: eventTime(event), - }); - span.log({ - ...(input !== undefined ? { input } : {}), + context, metadata, - }); - - turn.stepsByIndex.set(event.data.stepIndex, { - ...(input !== undefined ? { input } : {}), + ); + const ids = await generateEveIds("turn", event.sessionId, event.turnId); + return { + key, metadata, - metrics: {}, - ...(output !== undefined ? { output } : {}), - reasoning, + rootSpanId: span.rootSpanId, + rowId: ids.rowId, span, - }); + spanId: span.spanId, + }; } - private handleReasoningCompleted( - event: Extract< - EveHandleMessageStreamEvent, - { type: "reasoning.completed" } - >, - ctx: unknown, - ): void { - const sessionId = sessionIdFromContext(ctx); - if (!sessionId) { - return; + private async turnForScope( + scope: EveInstrumentationModelCallStartedEvent["scope"], + context: EveInstrumentationHandlerContext, + ): Promise { + const key = turnKey(scope.sessionId, scope.turnId); + const existing = this.turnsByKey.get(key); + if (existing !== undefined) { + return existing; } - - const reasoning = storeEveReasoning(this.state, sessionId, event); - const step = this.stepForEvent(event, ctx); - if (step) { - step.reasoning = reasoning; - step.output = mergeEveReasoning(step.output, reasoning); + const starting = this.turnsStartingByKey.get(key); + if (starting !== undefined) { + return await starting; } + + const ids = await generateEveIds("turn", scope.sessionId, scope.turnId); + const persistedRootSpanId = readSpanReference( + context.state.get(), + )?.rootSpanId; + const rememberedRootSpanId = eveRootSpanIdsByTurnKey.get(key); + return { + key, + metadata: this.metadata(scope.sessionId), + rootSpanId: + persistedRootSpanId ?? + rememberedRootSpanId ?? + (await deterministicEveId( + "eve:root", + scope.rootSessionId ?? scope.sessionId, + scope.turnId, + )), + rowId: ids.rowId, + span: NOOP_SPAN, + spanId: ids.spanId, + }; } - private handleMessageCompleted( - event: Extract, - ctx: unknown, + private updateTurn( + turn: TurnState, + event: { input?: unknown; output?: unknown }, ): void { - const step = this.stepForEvent(event, ctx); - if (!step) { + currentLogger()?.updateSpan({ + id: turn.rowId, + root_span_id: turn.rootSpanId, + span_id: turn.spanId, + ...event, + }); + } + + private closeOpenModel(attemptId: string, idempotencyKey: string): void { + const openModels = this.openModelsByAttempt.get(attemptId); + if (openModels === undefined) { return; } - - const existingMessage = eveOutputMessage(step.output); - const existingToolCalls = isObject(existingMessage) - ? existingMessage.tool_calls - : undefined; - step.output = mergeEveReasoning( - [ - { - finish_reason: normalizedFinishReason(event.data.finishReason), - index: 0, - message: { - content: event.data.message, - role: "assistant", - ...(Array.isArray(existingToolCalls) - ? { tool_calls: existingToolCalls } - : {}), - }, - }, - ], - step.reasoning, - ); - - const turn = this.turnForEvent(event, ctx); - if (turn && event.data.finishReason !== "tool-calls") { - turn.output = event.data.message; + openModels.delete(idempotencyKey); + if (openModels.size === 0) { + this.openModelsByAttempt.delete(attemptId); } } - private handleResultCompleted( - event: Extract, - ctx: unknown, + private restoreTurnMetricContributions( + turnKey: string, + state: unknown, ): void { - const step = this.stepForEvent(event, ctx); - if (step) { - step.output = mergeEveReasoning( - [ - { - finish_reason: "stop", - index: 0, - message: { - content: event.data.result, - role: "assistant", - }, - }, - ], - step.reasoning, - ); - } - - const turn = this.turnForEvent(event, ctx); - if (turn) { - turn.output = event.data.result; + const persisted = readSpanReference(state)?.turnMetricContributions; + if ( + persisted === undefined || + this.turnMetricContributionsByKey.has(turnKey) + ) { + return; } + this.turnMetricContributionsByKey.set(turnKey, { ...persisted }); } - private async handleActionsRequested( - event: Extract, - ctx: unknown, - hookMetadata?: Record, - ): Promise { - const turn = await this.ensureTurn(event, ctx, hookMetadata); - const sessionId = sessionIdFromContext(ctx); - if (!turn || !sessionId) { - return; + private updateTurnMetrics( + turn: TurnState, + idempotencyKey: string, + metrics: Record, + ): void { + const contributions = { + ...(this.turnMetricContributionsByKey.get(turn.key) ?? {}), + [idempotencyKey]: metrics, + }; + this.turnMetricContributionsByKey.set(turn.key, contributions); + const totals: Record = {}; + for (const contribution of Object.values(contributions)) { + for (const [name, value] of Object.entries(contribution)) { + totals[name] = (totals[name] ?? 0) + value; + } } - - const traceActions = event.data.actions.filter(isTraceableActionRequest); - if (traceActions.length === 0) { + if (Object.keys(totals).length === 0) { return; } - - for (const action of traceActions) { - if (isToolCallAction(action)) { - await this.startRequestedTool(event, turn, sessionId, action); - } else if (isLocalSubagentCallAction(action)) { - await this.startRequestedSubagent(event, turn, sessionId, action); - } - } - - const step = turn.stepsByIndex.get(event.data.stepIndex); - if (!step) { - return; - } - - const toolCallsById = new Map(); - if (Array.isArray(step.output) && isObject(step.output[0])) { - const message = step.output[0]["message"]; - if (isObject(message) && Array.isArray(message["tool_calls"])) { - for (const toolCall of message["tool_calls"]) { - if (isObject(toolCall) && typeof toolCall["id"] === "string") { - toolCallsById.set(toolCall["id"], toolCall); - } - } - } - } - for (const action of traceActions) { - const name = - action.kind === "tool-call" - ? action.toolName - : (action.subagentName ?? action.name ?? "agent"); - toolCallsById.set(action.callId, { - function: { - arguments: JSON.stringify(action.input), - name, - }, - id: action.callId, - type: "function", - }); - } - - step.output = mergeEveReasoning( - [ - { - finish_reason: "tool_calls", - index: 0, - message: { - content: null, - role: "assistant", - tool_calls: [...toolCallsById.values()], - }, - }, - ], - step.reasoning, - ); - } - - private async handleActionResult( - event: Extract, - ctx: unknown, - hookMetadata?: Record, - ): Promise { - if (isToolResult(event.data.result)) { - await this.handleToolResult(event, ctx, event.data.result, hookMetadata); - return; - } - if (isSubagentResult(event.data.result)) { - await this.handleSubagentResult( - event, - ctx, - event.data.result, - hookMetadata, - ); - } - } - - private async handleToolResult( - event: Extract, - ctx: unknown, - result: EveRuntimeToolResultActionResult, - hookMetadata?: Record, - ): Promise { - const sessionId = sessionIdFromContext(ctx); - if (!sessionId) { - return; - } - - const key = toolKey(sessionId, result.callId); - if (this.completedToolKeys.has(key)) { - return; - } - const tool = - this.toolsByCallKey.get(key) ?? - (await this.startSyntheticTool(event, ctx, result, hookMetadata)); - if (!tool) { - return; - } - const failed = - event.data.status === "failed" || - result.isError === true || - event.data.error !== undefined; - tool.span.log({ - ...(failed - ? { - error: actionResultError(event.data.error, result.output), - } - : {}), - metadata: tool.metadata, - output: result.output, + currentLogger()?.updateSpan({ + id: turn.rowId, + metrics: totals, + root_span_id: turn.rootSpanId, + span_id: turn.spanId, }); - - const endTime = eventTime(event); - tool.span.end(endTime === undefined ? undefined : { endTime }); - this.toolsByCallKey.delete(key); - this.completedToolKeys.set(key, true); } - private async handleSubagentCalled( - event: Extract, - ctx: unknown, - hookMetadata?: Record, - ): Promise { - if (event.data.remote?.url) { - return; - } - - const turn = await this.ensureTurn(event, ctx, hookMetadata); - const sessionId = sessionIdFromContext(ctx); - if (!turn || !sessionId) { - return; - } - - const key = toolKey(sessionId, event.data.callId); - const metadata = toolMetadataFromTurn(turn); - const existing = this.toolsByCallKey.get(key); - if (existing) { - existing.metadata = { ...existing.metadata, ...metadata }; - existing.span.log({ metadata: existing.metadata }); - return; - } - if (this.completedToolKeys.has(key)) { - return; - } - - const { rowId: eventId, spanId } = await generateEveIds( - "subagent", - sessionId, - event.data.callId, - ); - const pending = this.toolsByCallKey.get(key); - if (pending || this.completedToolKeys.has(key)) { - if (pending) { - pending.metadata = { ...pending.metadata, ...metadata }; - pending.span.log({ metadata: pending.metadata }); - } - return; - } - const span = await this.startEveChildSpan(turn.span, { + private async resumeModelSpan( + event: + | EveInstrumentationModelCallCompletedEvent + | EveInstrumentationModelCallFailedEvent, + context: EveInstrumentationHandlerContext, + turn: TurnState, + ): Promise { + const ids = await generateEveIds("step", event.idempotencyKey); + return await this.startEveSpan(context, { event: { - id: eventId, - metadata, + id: ids.rowId, + metadata: this.metadata(event.scope.sessionId), }, - name: event.data.toolName ?? event.data.name, - spanAttributes: { type: SpanTypeAttribute.TOOL }, - spanId, - startTime: eventTime(event), - }); - span.log({ metadata }); - this.toolsByCallKey.set(key, { - metadata, - span, - turnKey: turnKey(sessionId, event.data.turnId), + name: "eve.step", + parentSpanIds: { + rootSpanId: turn.rootSpanId, + spanId: turn.spanId, + }, + spanAttributes: { type: SpanTypeAttribute.LLM }, + spanId: ids.spanId, }); } - private async handleSubagentCompleted( - event: Extract, - ctx: unknown, - hookMetadata?: Record, - ): Promise { - const sessionId = sessionIdFromContext(ctx); - if (!sessionId) { - return; - } - - const key = toolKey(sessionId, event.data.callId); - if (this.completedToolKeys.has(key)) { - return; - } - const subagent = - this.toolsByCallKey.get(key) ?? - (await this.startSyntheticSubagent(event, ctx, hookMetadata)); - if (!subagent) { - return; - } - subagent.span.log({ - ...(event.data.status === "failed" - ? { - error: actionResultError(event.data.error, event.data.output), - } - : {}), - metadata: subagent.metadata, - ...(event.data.output !== undefined ? { output: event.data.output } : {}), - }); - const endTime = eventTime(event); - const recordedEndTime = subagent.span.end( - endTime === undefined ? undefined : { endTime }, + private async resumeActionSpan( + event: + | EveInstrumentationActionCompletedEvent + | EveInstrumentationActionFailedEvent, + context: EveInstrumentationHandlerContext, + ): Promise { + const turn = await this.turnForScope(event.scope, context); + const separator = event.idempotencyKey.lastIndexOf(":"); + const callId = + separator < 0 + ? event.idempotencyKey + : event.idempotencyKey.slice(separator + 1); + const ids = await generateEveIds( + "action", + event.scope.sessionId, + event.scope.turnId, + callId, ); - this.state.update((current) => { - const normalized = normalizeEveTraceState(current); - return { - ...normalized, - spanReferences: normalized.spanReferences.map((reference) => - reference.spanId === subagent.span.spanId - ? { ...reference, endTime: recordedEndTime } - : reference, - ), - }; - }); - } - - private async handleSubagentResult( - event: Extract, - ctx: unknown, - result: Extract, - hookMetadata?: Record, - ): Promise { - const sessionId = sessionIdFromContext(ctx); - if (!sessionId) { - return; - } - - const key = toolKey(sessionId, result.callId); - if (this.completedToolKeys.has(key)) { - return; - } - const subagent = - this.toolsByCallKey.get(key) ?? - (await this.startSyntheticSubagentResult( - event, - ctx, - result, - hookMetadata, - )); - if (!subagent) { - return; - } - const isError = - event.data.status === "failed" || - result.isError === true || - event.data.error !== undefined; - subagent.span.log({ - ...(isError - ? { - error: actionResultError(event.data.error, result.output), - } - : {}), - metadata: subagent.metadata, - output: result.output, + return await this.startEveSpan(context, { + event: { id: ids.rowId, metadata: turn.metadata }, + name: "eve.action", + parentSpanIds: { + rootSpanId: turn.rootSpanId, + spanId: turn.spanId, + }, + spanAttributes: { type: SpanTypeAttribute.TOOL }, + spanId: ids.spanId, }); - const endTime = eventTime(event); - subagent.span.end(endTime === undefined ? undefined : { endTime }); - - this.toolsByCallKey.delete(key); - this.completedToolKeys.set(key, true); } - private handleStepCompleted( - event: Extract, - ctx: unknown, - ): void { - const step = this.stepForEvent(event, ctx); - if (!step) { - return; - } - - const usage = event.data.usage; - const inputTokens = - typeof usage?.inputTokens === "number" && - Number.isFinite(usage.inputTokens) && - usage.inputTokens >= 0 - ? usage.inputTokens - : undefined; - const outputTokens = - typeof usage?.outputTokens === "number" && - Number.isFinite(usage.outputTokens) && - usage.outputTokens >= 0 - ? usage.outputTokens - : undefined; - const cacheReadTokens = - typeof usage?.cacheReadTokens === "number" && - Number.isFinite(usage.cacheReadTokens) && - usage.cacheReadTokens >= 0 - ? usage.cacheReadTokens - : undefined; - const cacheWriteTokens = - typeof usage?.cacheWriteTokens === "number" && - Number.isFinite(usage.cacheWriteTokens) && - usage.cacheWriteTokens >= 0 - ? usage.cacheWriteTokens - : undefined; - const costUsd = - typeof usage?.costUsd === "number" && - Number.isFinite(usage.costUsd) && - usage.costUsd >= 0 - ? usage.costUsd - : undefined; - const total = - inputTokens !== undefined && outputTokens !== undefined - ? inputTokens + outputTokens - : undefined; - const metrics = { - ...(inputTokens !== undefined ? { prompt_tokens: inputTokens } : {}), - ...(outputTokens !== undefined - ? { completion_tokens: outputTokens } - : {}), - ...(total !== undefined ? { tokens: total } : {}), - ...(cacheReadTokens !== undefined - ? { prompt_cached_tokens: cacheReadTokens } - : {}), - ...(cacheWriteTokens !== undefined - ? { prompt_cache_creation_tokens: cacheWriteTokens } - : {}), - ...(costUsd !== undefined ? { estimated_cost: costUsd } : {}), + private metadata(sessionId: string): Record { + return { + ...(this.configuredMetadata ?? {}), + "eve.session_id": sessionId, }; - step.metrics = { ...step.metrics, ...metrics }; - const sessionId = sessionIdFromContext(ctx); - if (Array.isArray(step.output) && isObject(step.output[0])) { - const finishReason = step.output[0].finish_reason; - if (typeof finishReason !== "string") { - step.output[0].finish_reason = normalizedFinishReason( - event.data.finishReason, - ); - } - } - step.span.log({ - ...(step.input !== undefined ? { input: step.input } : {}), - metadata: step.metadata, - metrics, - output: step.output, - }); - const endTime = eventTime(event); - step.span.end(endTime === undefined ? undefined : { endTime }); - - const turn = this.turnForEvent(event, ctx); - if (turn) { - for (const [key, value] of Object.entries(metrics)) { - turn.metrics[key] = (turn.metrics[key] ?? 0) + value; - } - turn.stepsByIndex.delete(event.data.stepIndex); - } - this.markStepEnded(event.data.turnId, event.data.stepIndex); - if (sessionId) { - clearStoredEveReasoning( - this.state, - sessionId, - event.data.turnId, - event.data.stepIndex, - ); - } - } - - private handleStepFailed( - event: Extract, - ctx: unknown, - ): void { - const step = this.stepForEvent(event, ctx); - if (step) { - step.span.log({ - error: errorFromMessage( - event.data.message, - event.data.code, - event.data.details, - ), - }); - const endTime = eventTime(event); - step.span.end(endTime === undefined ? undefined : { endTime }); - } - - const turn = this.turnForEvent(event, ctx); - turn?.stepsByIndex.delete(event.data.stepIndex); - this.markStepEnded(event.data.turnId, event.data.stepIndex); - const sessionId = sessionIdFromContext(ctx); - if (sessionId) { - clearStoredEveReasoning( - this.state, - sessionId, - event.data.turnId, - event.data.stepIndex, - ); - } } +} - private handleTurnCompleted( - event: Extract, - ctx: unknown, - ): void { - const turn = this.turnForEvent(event, ctx); - if (!turn) { - return; - } - - this.finalizeTurn(turn, { - endTime: eventTime(event), - }); +function readSpanReference(value: unknown): EveSpanReference | undefined { + if (!isObject(value)) { + return undefined; } - - private handleTurnFailed( - event: Extract, - ctx: unknown, - ): void { - const turn = this.turnForEvent(event, ctx); - if (!turn) { - return; - } - - this.finalizeTurn(turn, { - endTime: eventTime(event), - error: errorFromMessage( - event.data.message, - event.data.code, - event.data.details, - ), - }); + const exported = value["exported"]; + const rootSpanId = value["rootSpanId"]; + const spanId = value["spanId"]; + if ( + typeof exported !== "string" || + typeof rootSpanId !== "string" || + typeof spanId !== "string" + ) { + return undefined; } - - private handleSessionFailed( - event: Extract, - ctx: unknown, - ): void { - const sessionId = event.data.sessionId || sessionIdFromContext(ctx); - if (!sessionId) { - return; - } - const error = errorFromMessage( - event.data.message, - event.data.code, - event.data.details, - ); - for (const [key, turn] of this.turnsByKey) { - if (!key.startsWith(`${sessionId}:`)) { - continue; - } - this.finalizeTurn(turn, { - endTime: eventTime(event), - error, - }); - } - - for (const [key, tool] of this.toolsByCallKey) { - if (key.startsWith(`${sessionId}:`)) { - const endTime = eventTime(event); - if (!tool.endedByTurn) { - tool.span.log({ metadata: tool.metadata }); - tool.span.end(endTime === undefined ? undefined : { endTime }); - tool.endedByTurn = true; + return { + exported, + rootSpanId, + spanId, + ...(isObject(value["turnMetricContributions"]) + ? { + turnMetricContributions: numericMetricContributions( + value["turnMetricContributions"], + ), } - } - } - } - - private handleSessionCompleted( - event: Extract, - ctx: unknown, - ): void { - const sessionId = sessionIdFromContext(ctx); - if (!sessionId) { - return; - } - for (const [key, turn] of this.turnsByKey) { - if (!key.startsWith(`${sessionId}:`)) { - continue; - } - this.finalizeTurn(turn, { - endTime: eventTime(event), - }); - } + : {}), + }; +} - for (const [key, tool] of this.toolsByCallKey) { - if (key.startsWith(`${sessionId}:`) && !tool.endedByTurn) { - const endTime = eventTime(event); - tool.span.log({ metadata: tool.metadata }); - tool.span.end(endTime === undefined ? undefined : { endTime }); - tool.endedByTurn = true; - } - } +function capturedModelInput( + input: EveInstrumentationModelInput | undefined, +): unknown[] | undefined { + if (input === undefined) { + return undefined; } - - private async ensureTurn( - event: Extract< - EveHandleMessageStreamEvent, - { - data: { readonly sequence: number; readonly turnId: string }; + const value: unknown[] = []; + if (typeof input.instructions === "string") { + value.push({ content: input.instructions, role: "system" }); + } else if (Array.isArray(input.instructions)) { + for (const instruction of input.instructions) { + const projected = projectedModelMessage(instruction); + if (projected?.role === "system") { + value.push(projected); } - >, - ctx: unknown, - hookMetadata?: Record, - ): Promise { - const sessionId = sessionIdFromContext(ctx); - if (!sessionId) { - return undefined; } - - const key = turnKey(sessionId, event.data.turnId); - const existing = this.turnsByKey.get(key); - if (existing) { - return existing; + } else if (isObject(input.instructions)) { + const instruction = projectedModelMessage(input.instructions); + if (instruction?.role === "system") { + value.push(instruction); } - - const metadata = { - ...readEveTraceState(this.state).metadata, - ...(hookMetadata ?? {}), - "eve.session_id": sessionId, - }; - const span = await this.startTurnSpan(sessionId, event, ctx, metadata); - span.log({ metadata }); - const state = { - key, - metadata, - metrics: {}, - sessionId, - span, - stepsByIndex: new Map(), - turnId: event.data.turnId, - }; - this.turnsByKey.set(key, state); - return state; } - - private async startRequestedTool( - event: Extract, - turn: TurnState, - sessionId: string, - action: EveRuntimeToolCallActionRequest, - ): Promise { - const key = toolKey(sessionId, action.callId); - if (this.toolsByCallKey.has(key) || this.completedToolKeys.has(key)) { - return; + for (const message of input.messages) { + const projected = projectedModelMessage(message); + if (projected !== undefined) { + value.push(projected); } - - const metadata = toolMetadataFromTurn(turn); - const { rowId: eventId, spanId } = await generateEveIds( - "tool", - sessionId, - event.data.turnId, - action.callId, - ); - if (this.toolsByCallKey.has(key) || this.completedToolKeys.has(key)) { - return; - } - const span = await this.startEveChildSpan(turn.span, { - event: { - id: eventId, - input: action.input, - metadata, - }, - name: action.toolName, - spanAttributes: { type: SpanTypeAttribute.TOOL }, - spanId, - startTime: eventTime(event), - }); - span.log({ input: action.input, metadata }); - this.toolsByCallKey.set(key, { - metadata, - span, - turnKey: turnKey(sessionId, event.data.turnId), - }); } + return value; +} - private async startRequestedSubagent( - event: Extract, - turn: TurnState, - sessionId: string, - action: Extract, - ): Promise { - const key = toolKey(sessionId, action.callId); - if (this.toolsByCallKey.has(key) || this.completedToolKeys.has(key)) { - return; - } - - const name = action.subagentName ?? action.name ?? "agent"; - const metadata = toolMetadataFromTurn(turn); - const { rowId: eventId, spanId } = await generateEveIds( - "subagent", - sessionId, - action.callId, - ); - if (this.toolsByCallKey.has(key) || this.completedToolKeys.has(key)) { - return; - } - const span = await this.startEveChildSpan(turn.span, { - event: { - id: eventId, - input: action.input, - metadata, - }, - name, - spanAttributes: { type: SpanTypeAttribute.TOOL }, - spanId, - startTime: eventTime(event), - }); - span.log({ input: action.input, metadata }); - this.toolsByCallKey.set(key, { - metadata, - span, - turnKey: turnKey(sessionId, event.data.turnId), - }); - } - - private async startSyntheticTool( - event: Extract, - ctx: unknown, - result: EveRuntimeToolResultActionResult, - hookMetadata?: Record, - ): Promise { - const turn = await this.ensureTurn(event, ctx, hookMetadata); - const sessionId = sessionIdFromContext(ctx); - if (!turn || !sessionId) { - return undefined; - } - - const metadata = toolMetadataFromTurn(turn); - const { rowId: eventId, spanId } = await generateEveIds( - "tool", - sessionId, - event.data.turnId, - result.callId, - ); - const existing = this.toolsByCallKey.get(toolKey(sessionId, result.callId)); - if (existing) { - return existing; - } - const span = await this.startEveChildSpan(turn.span, { - event: { - id: eventId, - metadata, - }, - name: result.toolName, - spanAttributes: { type: SpanTypeAttribute.TOOL }, - spanId, - startTime: eventTime(event), - }); - span.log({ metadata }); - const state = { - metadata, - span, - turnKey: turnKey(sessionId, event.data.turnId), - }; - this.toolsByCallKey.set(toolKey(sessionId, result.callId), state); - return state; - } - - private async startSyntheticSubagent( - event: Extract, - ctx: unknown, - hookMetadata?: Record, - ): Promise { - const turn = await this.ensureTurn(event, ctx, hookMetadata); - const sessionId = sessionIdFromContext(ctx); - if (!turn || !sessionId) { - return undefined; - } - - const metadata = toolMetadataFromTurn(turn); - const { rowId: eventId, spanId } = await generateEveIds( - "subagent", - sessionId, - event.data.callId, - ); - const existing = this.toolsByCallKey.get( - toolKey(sessionId, event.data.callId), - ); - if (existing) { - return existing; - } - const span = await this.startEveChildSpan(turn.span, { - event: { - id: eventId, - metadata, - }, - name: event.data.subagentName, - spanAttributes: { type: SpanTypeAttribute.TOOL }, - spanId, - startTime: eventTime(event), - }); - span.log({ metadata }); - const state = { - metadata, - span, - turnKey: turnKey(sessionId, event.data.turnId), - }; - this.toolsByCallKey.set(toolKey(sessionId, event.data.callId), state); - return state; - } - - private async startSyntheticSubagentResult( - event: Extract, - ctx: unknown, - result: Extract, - hookMetadata?: Record, - ): Promise { - const turn = await this.ensureTurn(event, ctx, hookMetadata); - const sessionId = sessionIdFromContext(ctx); - if (!turn || !sessionId) { - return undefined; - } - - const metadata = toolMetadataFromTurn(turn); - const { rowId: eventId, spanId } = await generateEveIds( - "subagent", - sessionId, - result.callId, - ); - const existing = this.toolsByCallKey.get(toolKey(sessionId, result.callId)); - if (existing) { - return existing; - } - const span = await this.startEveChildSpan(turn.span, { - event: { - id: eventId, - metadata, - }, - name: result.subagentName, - spanAttributes: { type: SpanTypeAttribute.TOOL }, - spanId, - startTime: eventTime(event), - }); - span.log({ metadata }); - const state = { - metadata, - span, - turnKey: turnKey(sessionId, event.data.turnId), - }; - this.toolsByCallKey.set(toolKey(sessionId, result.callId), state); - return state; +function projectedModelMessage( + value: unknown, +): { content: unknown; role: string } | undefined { + if (!isObject(value)) { + return undefined; } - - private async startTurnSpan( - sessionId: string, - event: Extract< - EveHandleMessageStreamEvent, - { data: { readonly sequence: number; readonly turnId: string } } - >, - ctx: unknown, - metadata: Record, - ): Promise { - const session = isObject(ctx) ? ctx["session"] : undefined; - const parent = isObject(session) ? session["parent"] : undefined; - const parentTurn = isObject(parent) ? parent["turn"] : undefined; - const parentLineage = - isObject(parent) && - typeof parent["callId"] === "string" && - typeof parent["sessionId"] === "string" && - isObject(parentTurn) && - typeof parentTurn["id"] === "string" - ? { - callId: parent["callId"], - sessionId: parent["sessionId"], - turnId: parentTurn["id"], - } - : undefined; - const [{ rowId: eventId, spanId }, rootSpanId, parentSpanId] = - await Promise.all([ - generateEveIds("turn", sessionId, event.data.turnId), - deterministicEveId( - "eve:root", - parentLineage?.sessionId ?? sessionId, - parentLineage?.turnId ?? event.data.turnId, - ), - parentLineage - ? deterministicEveId( - "eve:subagent", - parentLineage.sessionId, - parentLineage.callId, - ) - : Promise.resolve(undefined), - ]); - - return await this.startEveSpan({ - event: { - id: eventId, - metadata, - }, - name: "eve.turn", - parentSpanIds: parentSpanId - ? { rootSpanId, spanId: parentSpanId } - : { parentSpanIds: [], rootSpanId }, - spanAttributes: { type: SpanTypeAttribute.TASK }, - spanId, - startTime: eventTime(event), - }); + const role = value["role"]; + if ( + role !== "system" && + role !== "user" && + role !== "assistant" && + role !== "tool" + ) { + return undefined; } - - private turnForEvent( - event: Extract< - EveHandleMessageStreamEvent, - { data: { readonly turnId: string } } - >, - ctx: unknown, - ): TurnState | undefined { - const sessionId = sessionIdFromContext(ctx); - return sessionId - ? this.turnsByKey.get(turnKey(sessionId, event.data.turnId)) - : undefined; + const content = value["content"]; + if (typeof content === "string") { + return { content, role }; } - - private stepForEvent( - event: Extract< - EveHandleMessageStreamEvent, - { data: { readonly stepIndex: number; readonly turnId: string } } - >, - ctx: unknown, - ): StepState | undefined { - return this.turnForEvent(event, ctx)?.stepsByIndex.get( - event.data.stepIndex, - ); + if (!Array.isArray(content)) { + return undefined; } - - private finalizeTurn( - turn: TurnState, - args: { endTime: number | undefined; error?: Error }, - ): void { - const { endTime } = args; - for (const step of turn.stepsByIndex.values()) { - step.span.log({ - ...(step.input !== undefined ? { input: step.input } : {}), - metadata: step.metadata, - metrics: step.metrics, - output: step.output, - }); - step.span.end(endTime === undefined ? undefined : { endTime }); - } - turn.stepsByIndex.clear(); - - for (const tool of this.toolsByCallKey.values()) { - if (tool.turnKey !== turn.key) { - continue; + const projected: unknown[] = []; + for (const part of content) { + if (!isObject(part) || typeof part["type"] !== "string") { + continue; + } + switch (part["type"]) { + case "text": + case "reasoning": + if (typeof part["text"] === "string") { + projected.push({ text: part["text"], type: part["type"] }); + } + break; + case "image": { + if (part["image"] === undefined) { + break; + } + const mediaType = + typeof part["mediaType"] === "string" + ? part["mediaType"] + : "image/png"; + projected.push({ + image: mediaAttachment(part["image"], mediaType, undefined), + ...(typeof part["mediaType"] === "string" ? { mediaType } : {}), + type: "image", + }); + break; } - if (tool.endedByTurn) { - continue; + case "file": { + const mediaType = + typeof part["mediaType"] === "string" + ? part["mediaType"] + : "application/octet-stream"; + const data = + part["data"] === undefined + ? undefined + : mediaAttachment( + part["data"], + mediaType, + typeof part["filename"] === "string" + ? part["filename"] + : undefined, + ); + projected.push({ + ...(data === undefined ? {} : { data }), + ...(typeof part["filename"] === "string" + ? { filename: part["filename"] } + : {}), + mediaType, + type: "file", + }); + break; } - tool.span.log({ metadata: tool.metadata }); - tool.span.end(endTime === undefined ? undefined : { endTime }); - tool.endedByTurn = true; + case "tool-call": + if ( + typeof part["toolCallId"] === "string" && + typeof part["toolName"] === "string" + ) { + projected.push({ + input: part["input"], + toolCallId: part["toolCallId"], + toolName: part["toolName"], + type: "tool-call", + }); + } + break; + case "tool-result": + if ( + typeof part["toolCallId"] === "string" && + typeof part["toolName"] === "string" + ) { + projected.push({ + output: projectedToolOutput(part["output"]), + toolCallId: part["toolCallId"], + toolName: part["toolName"], + type: "tool-result", + }); + } + break; + case "tool-approval-response": + if ( + typeof part["approvalId"] === "string" && + typeof part["approved"] === "boolean" + ) { + projected.push({ + approvalId: part["approvalId"], + approved: part["approved"], + ...(typeof part["reason"] === "string" + ? { reason: part["reason"] } + : {}), + type: "tool-approval-response", + }); + } + break; + default: + break; } + } + return { content: projected, role }; +} - if (args.error) { - turn.span.log({ error: args.error }); - } else { - turn.span.log({ - metadata: turn.metadata, - metrics: turn.metrics, - output: turn.output, - }); - } - turn.span.end(endTime === undefined ? undefined : { endTime }); - this.turnsByKey.delete(turn.key); - this.state.update((current) => { - const normalized = normalizeEveTraceState(current); +function projectedToolOutput(value: unknown): unknown { + if (!isObject(value)) { + return value; + } + if (typeof value["type"] !== "string") { + return undefined; + } + switch (value["type"]) { + case "text": + case "json": + case "error-text": + case "error-json": + return { type: value["type"], value: value["value"] }; + case "execution-denied": return { - ...normalized, - reasoningBlocks: normalized.reasoningBlocks.filter( - (entry) => - !entry.key.startsWith(`${turn.sessionId}\0${turn.turnId}\0`), - ), - stepStarts: normalized.stepStarts.filter( - (entry) => entry.turnId !== turn.turnId, - ), + ...(typeof value["reason"] === "string" + ? { reason: value["reason"] } + : {}), + type: "execution-denied", }; - }); - } - - private cleanupSession(sessionId: string): void { - const keyPrefix = `${sessionId}:`; - for (const key of this.turnsByKey.keys()) { - if (key.startsWith(keyPrefix)) { - this.turnsByKey.delete(key); - } - } - for (const key of this.toolsByCallKey.keys()) { - if (key.startsWith(keyPrefix)) { - this.toolsByCallKey.delete(key); + case "content": { + if (!Array.isArray(value["value"])) { + return { type: "content", value: [] }; } - } - for (const key of this.completedToolKeys.keys()) { - if (key.startsWith(keyPrefix)) { - this.completedToolKeys.delete(key); + const content: unknown[] = []; + for (const part of value["value"]) { + if (!isObject(part) || typeof part["type"] !== "string") { + continue; + } + if (part["type"] === "text" && typeof part["text"] === "string") { + content.push({ text: part["text"], type: "text" }); + } else if ( + (part["type"] === "file-data" || part["type"] === "image-data") && + part["data"] !== undefined && + typeof part["mediaType"] === "string" + ) { + const data = mediaAttachment( + part["data"], + part["mediaType"], + typeof part["filename"] === "string" ? part["filename"] : undefined, + ); + content.push({ + ...(data === undefined ? {} : { data }), + ...(typeof part["filename"] === "string" + ? { filename: part["filename"] } + : {}), + mediaType: part["mediaType"], + type: part["type"], + }); + } else if ( + (part["type"] === "file-url" || part["type"] === "image-url") && + typeof part["url"] === "string" + ) { + content.push({ type: part["type"], url: part["url"] }); + } } + return { + type: "content", + value: content, + }; } - this.state.update(() => emptyEveTraceState()); - } - - private async flushInstrumentation(): Promise { - try { - await flush(); - return true; - } catch (error) { - debugLogger.warn("Error in Eve flush instrumentation:", error); - return false; - } - } -} - -function emptyEveTraceState(): EveTraceState { - return { - llmInputs: [], - metadata: {}, - reasoningBlocks: [], - spanReferences: [], - stepStarts: [], - }; -} - -function normalizeEveTraceState(state: unknown): EveTraceState { - if (!isObject(state)) { - return emptyEveTraceState(); + default: + return undefined; } - const metadata = isObject(state["metadata"]) ? state["metadata"] : {}; - const spanReferences = Array.isArray(state["spanReferences"]) - ? state["spanReferences"] - .flatMap((entry): EveTraceState["spanReferences"] => { - if (!isObject(entry)) { - return []; - } - const exported = entry["exported"]; - const endTime = entry["endTime"]; - const rootSpanId = entry["rootSpanId"]; - const rowId = entry["rowId"]; - const spanId = entry["spanId"]; - const startEvent = entry["startEvent"]; - const startEventCreated = isObject(startEvent) - ? startEvent["created"] - : undefined; - const startEventMetrics = isObject(startEvent) - ? startEvent["metrics"] - : undefined; - const startEventSpanAttributes = isObject(startEvent) - ? startEvent["span_attributes"] - : undefined; - const startEventSpanParents = isObject(startEvent) - ? startEvent["span_parents"] - : undefined; - const normalizedStartEvent = - typeof startEventCreated === "string" && - isObject(startEventMetrics) && - typeof startEventMetrics["start"] === "number" && - Number.isFinite(startEventMetrics["start"]) && - isObject(startEventSpanAttributes) && - Array.isArray(startEventSpanParents) && - startEventSpanParents.every( - (parent): parent is string => typeof parent === "string", - ) - ? { - created: startEventCreated, - metrics: { start: startEventMetrics["start"] }, - span_attributes: { ...startEventSpanAttributes }, - span_parents: [...startEventSpanParents], - } - : undefined; - return typeof exported === "string" && - typeof rootSpanId === "string" && - typeof rowId === "string" && - typeof spanId === "string" - ? [ - { - ...(typeof endTime === "number" && Number.isFinite(endTime) - ? { endTime } - : {}), - exported, - rootSpanId, - rowId, - spanId, - ...(normalizedStartEvent - ? { startEvent: normalizedStartEvent } - : {}), - }, - ] - : []; - }) - .slice(-MAX_STORED_SPAN_REFERENCES) - : []; - const llmInputs = Array.isArray(state["llmInputs"]) - ? state["llmInputs"] - .flatMap((entry): EveTraceState["llmInputs"] => { - if (!isObject(entry)) { - return []; - } - const key = entry["key"]; - const input = entry["input"]; - return typeof key === "string" && isCapturedModelInput(input) - ? [{ input, key }] - : []; - }) - .slice(-MAX_STORED_LLM_INPUTS) - : []; - const reasoningBlocks = Array.isArray(state["reasoningBlocks"]) - ? state["reasoningBlocks"] - .flatMap((entry): EveTraceState["reasoningBlocks"] => { - if (!isObject(entry)) { - return []; - } - const content = entry["content"]; - const eventAt = entry["eventAt"]; - const key = entry["key"]; - return typeof content === "string" && - (eventAt === undefined || typeof eventAt === "string") && - typeof key === "string" - ? [ - { - content, - ...(typeof eventAt === "string" ? { eventAt } : {}), - key, - }, - ] - : []; - }) - .slice(-MAX_STORED_REASONING_BLOCKS) - : []; - const stepStarts = Array.isArray(state["stepStarts"]) - ? state["stepStarts"] - .flatMap((entry): EveTraceState["stepStarts"] => { - if (!isObject(entry)) { - return []; - } - const ordinal = entry["ordinal"]; - const open = entry["open"]; - const stepIndex = entry["stepIndex"]; - const turnId = entry["turnId"]; - return typeof ordinal === "number" && - Number.isInteger(ordinal) && - ordinal >= 0 && - typeof open === "boolean" && - typeof stepIndex === "number" && - Number.isInteger(stepIndex) && - typeof turnId === "string" - ? [{ open, ordinal, stepIndex, turnId }] - : []; - }) - .slice(-MAX_STORED_STEP_STARTS) - : []; - return { - llmInputs, - metadata: { ...metadata }, - reasoningBlocks, - spanReferences, - stepStarts, - }; } -function readEveTraceState( - state: EveStateHandle, -): EveTraceState { - try { - return normalizeEveTraceState(state.get()); - } catch { - return emptyEveTraceState(); +function mediaAttachment( + data: unknown, + contentType: string, + filename: string | undefined, +): unknown { + const blob = convertDataToBlob(data, contentType); + if (blob === null) { + return data; } -} - -function storeEveReasoning( - state: EveStateHandle, - sessionId: string, - event: Extract, -): readonly EveReasoningBlock[] { - const eventAt = event.meta?.at; - const key = llmInputKey(sessionId, event.data.turnId, event.data.stepIndex); - let stored: readonly EveReasoningBlock[] = []; - state.update((current) => { - const normalized = normalizeEveTraceState(current); - const alreadyStored = normalized.reasoningBlocks.some( - (entry) => - entry.content === event.data.reasoning && - entry.eventAt === eventAt && - entry.key === key, - ); - const reasoningBlocks = alreadyStored - ? normalized.reasoningBlocks - : [ - ...normalized.reasoningBlocks, - { - content: event.data.reasoning, - ...(eventAt ? { eventAt } : {}), - key, - }, - ].slice(-MAX_STORED_REASONING_BLOCKS); - stored = reasoningBlocks.flatMap((entry) => - entry.key === key - ? [ - { - content: entry.content, - ...(entry.eventAt ? { eventAt: entry.eventAt } : {}), - }, - ] - : [], - ); - return alreadyStored ? normalized : { ...normalized, reasoningBlocks }; + return new Attachment({ + contentType, + data: blob, + filename: + filename ?? `attachment.${getExtensionFromMediaType(contentType)}`, }); - return stored; } -function readStoredEveReasoning( - state: EveStateHandle, - sessionId: string, - turnId: string, - stepIndex: number, -): readonly EveReasoningBlock[] { - const key = llmInputKey(sessionId, turnId, stepIndex); - return readEveTraceState(state).reasoningBlocks.flatMap((entry) => - entry.key === key - ? [ - { - content: entry.content, - ...(entry.eventAt ? { eventAt: entry.eventAt } : {}), - }, - ] - : [], - ); +function initialTurnInput(input: readonly unknown[]): unknown[] | undefined { + for (let index = input.length - 1; index >= 0; index--) { + const message = input[index]; + if (isObject(message) && message["role"] === "user") { + return [message]; + } + } + return undefined; } -function clearStoredEveReasoning( - state: EveStateHandle, - sessionId: string, - turnId: string, - stepIndex: number, -): void { - const key = llmInputKey(sessionId, turnId, stepIndex); - state.update((current) => { - const normalized = normalizeEveTraceState(current); - return { - ...normalized, - reasoningBlocks: normalized.reasoningBlocks.filter( - (entry) => entry.key !== key, - ), - }; +function modelOutput( + finishReason: string, + content: readonly EveInstrumentationContentPart[] | undefined, +): unknown { + const parts = content ?? []; + const text = parts + .filter( + ( + part, + ): part is Extract => + part.type === "text", + ) + .map((part) => part.text) + .join(""); + const reasoning = parts + .filter( + ( + part, + ): part is Extract< + EveInstrumentationContentPart, + { type: "reasoning" } + > => part.type === "reasoning", + ) + .map((part) => ({ content: part.text })); + const toolCalls = parts.flatMap((part) => { + if (part.type !== "tool-call") { + return []; + } + return [ + { + function: { + arguments: jsonString(part.input), + name: part.toolName, + }, + id: part.callId, + type: "function", + }, + ]; }); -} - -function eveOutputMessage(output: unknown): unknown { - return Array.isArray(output) && isObject(output[0]) - ? output[0]["message"] - : undefined; -} - -function mergeEveReasoning( - output: unknown, - reasoning: readonly { content: string }[], -): unknown | undefined { - if (reasoning.length === 0) { - return output; - } - - const choice = Array.isArray(output) && isObject(output[0]) ? output[0] : {}; - const message = isObject(choice["message"]) ? choice["message"] : {}; return [ { - ...choice, - index: typeof choice["index"] === "number" ? choice["index"] : 0, + finish_reason: normalizedFinishReason(finishReason), + index: 0, message: { - ...message, - content: "content" in message ? message["content"] : null, - reasoning: reasoning.map((block) => ({ content: block.content })), - role: - typeof message["role"] === "string" ? message["role"] : "assistant", + content: text.length === 0 ? null : text, + ...(reasoning.length === 0 ? {} : { reasoning }), + role: "assistant", + ...(toolCalls.length === 0 ? {} : { tool_calls: toolCalls }), }, }, ]; } -function captureEveModelInput( - state: EveStateHandle, - input: EveInstrumentationStepStartedEventInput, -): void { - const sessionId = input.session.id; - const turnId = input.turn.id; - const stepIndex = input.step.index; - const captured = capturedModelInput(input.modelInput); - if (!captured) { - return; - } - - const key = llmInputKey(sessionId, turnId, stepIndex); - state.update((current) => { - const normalized = normalizeEveTraceState(current); - const llmInputs = [...normalized.llmInputs, { input: captured, key }]; - return { - ...normalized, - llmInputs: llmInputs.slice(-MAX_STORED_LLM_INPUTS), - }; - }); -} - -function consumeCapturedEveModelInput( - state: EveStateHandle, - sessionId: string, - turnId: string, - stepIndex: number, -): CapturedEveModelInput | undefined { - try { - const key = llmInputKey(sessionId, turnId, stepIndex); - let input: CapturedEveModelInput | undefined; - state.update((current) => { - const normalized = normalizeEveTraceState(current); - const index = normalized.llmInputs.findIndex( - (candidate) => candidate.key === key, - ); - if (index < 0) { - return normalized; - } - input = normalized.llmInputs[index]?.input; - return { - ...normalized, - llmInputs: normalized.llmInputs.filter( - (_, candidateIndex) => candidateIndex !== index, - ), - }; - }); - return input; - } catch (error) { - debugLogger.warn("Error in Eve LLM input consumption:", error); - return undefined; - } +function finalText( + content: readonly EveInstrumentationContentPart[] | undefined, +): string | undefined { + const text = (content ?? []) + .filter( + ( + part, + ): part is Extract => + part.type === "text", + ) + .map((part) => part.text) + .join(""); + return text.length === 0 ? undefined : text; } -function capturedModelInput( - modelInput: EveInstrumentationModelInput, -): CapturedEveModelInput | undefined { - const { instructions, messages } = modelInput; - const value: CapturedEveModelMessage[] = []; - if (typeof instructions === "string") { - value.push({ content: instructions, role: "system" }); - } else if (instructions) { - value.push(...instructions.map(capturedEveModelMessage)); - } - value.push(...messages.map(capturedEveModelMessage)); - +function jsonString(value: unknown): string { try { - const cloned: unknown = JSON.parse(JSON.stringify(value)); - if (!Array.isArray(cloned)) { - return undefined; - } - return cloned; + return JSON.stringify(value) ?? "null"; } catch { - return undefined; + return "null"; } } -function capturedEveModelMessage( - message: EveModelMessage, -): CapturedEveModelMessage { - const { content, role } = message; - if (typeof content === "string") { - return { content, role }; +function normalizedFinishReason(finishReason: string): string { + if (finishReason === "content-filter") { + return "content_filter"; } - return { content: content.map(capturedEveModelContentPart), role }; -} - -function capturedEveModelContentPart( - part: EveModelMessageContentPart, -): Record { - switch (part.type) { - case "text": - case "reasoning": - return { text: part.text, type: part.type }; - case "image": - return { - image: part.image, - ...(part.mediaType !== undefined ? { mediaType: part.mediaType } : {}), - type: "image", - }; - case "file": - case "reasoning-file": - return { - data: part.data, - ...(part.type === "file" && part.filename !== undefined - ? { filename: part.filename } - : {}), - mediaType: part.mediaType, - type: part.type, - }; - case "custom": - return { - ...("kind" in part ? { kind: part.kind } : {}), - type: "custom", - }; - case "tool-call": - return { - input: part.input, - ...(part.providerExecuted !== undefined - ? { providerExecuted: part.providerExecuted } - : {}), - toolCallId: part.toolCallId, - toolName: part.toolName, - type: "tool-call", - }; - case "tool-result": { - const output = part.output; - let capturedOutput: Record; - switch (output.type) { - case "text": - case "error-text": - capturedOutput = { type: output.type, value: output.value }; - break; - case "json": - case "error-json": - capturedOutput = { type: output.type, value: output.value }; - break; - case "execution-denied": - capturedOutput = { - ...(output.reason !== undefined ? { reason: output.reason } : {}), - type: "execution-denied", - }; - break; - case "content": - capturedOutput = { - type: "content", - value: output.value.map(capturedEveModelContentPart), - }; - break; - } - return { - output: capturedOutput, - toolCallId: part.toolCallId, - toolName: part.toolName, - type: "tool-result", - }; - } - case "tool-approval-request": - return { - approvalId: part.approvalId, - ...(part.isAutomatic !== undefined - ? { isAutomatic: part.isAutomatic } - : {}), - ...(part.signature !== undefined ? { signature: part.signature } : {}), - toolCallId: part.toolCallId, - type: "tool-approval-request", - }; - case "tool-approval-response": - return { - approvalId: part.approvalId, - approved: part.approved, - ...(part.providerExecuted !== undefined - ? { providerExecuted: part.providerExecuted } - : {}), - ...(part.reason !== undefined ? { reason: part.reason } : {}), - type: "tool-approval-response", - }; - case "file-data": - case "image-data": - return { - data: part.data, - ...(part.type === "file-data" && part.filename !== undefined - ? { filename: part.filename } - : {}), - mediaType: part.mediaType, - type: part.type, - }; - case "file-url": - case "image-url": - return { - ...(part.type === "file-url" && part.mediaType !== undefined - ? { mediaType: part.mediaType } - : {}), - type: part.type, - url: part.url, - }; - case "file-id": - case "image-file-id": - return { fileId: part.fileId, type: part.type }; - case "file-reference": - case "image-file-reference": - return { - providerReference: part.providerReference, - type: part.type, - }; + if (finishReason === "tool-calls") { + return "tool_calls"; } + return finishReason; } -function isCapturedModelInput(input: unknown): input is CapturedEveModelInput { - return ( - Array.isArray(input) && - input.every( - (message) => - isObject(message) && - (message["role"] === "system" || - message["role"] === "user" || - message["role"] === "assistant" || - message["role"] === "tool") && - (typeof message["content"] === "string" || - (Array.isArray(message["content"]) && - message["content"].every(isObject))), - ) +function usageMetrics(usage: EveInstrumentationUsage): Record { + const inputTokens = validMetric(usage.inputTokens); + const outputTokens = validMetric(usage.outputTokens); + const cacheReadTokens = validMetric(usage.inputTokenDetails?.cacheReadTokens); + const cacheWriteTokens = validMetric( + usage.inputTokenDetails?.cacheWriteTokens, ); + return { + ...(inputTokens === undefined ? {} : { prompt_tokens: inputTokens }), + ...(outputTokens === undefined ? {} : { completion_tokens: outputTokens }), + ...(inputTokens === undefined || outputTokens === undefined + ? {} + : { tokens: inputTokens + outputTokens }), + ...(cacheReadTokens === undefined + ? {} + : { prompt_cached_tokens: cacheReadTokens }), + ...(cacheWriteTokens === undefined + ? {} + : { prompt_cache_creation_tokens: cacheWriteTokens }), + }; } -function llmInputKey( - sessionId: string, - turnId: string, - stepIndex: number, -): string { - return `${sessionId}\0${turnId}\0${stepIndex}`; +function validMetric(value: unknown): number | undefined { + return typeof value === "number" && Number.isFinite(value) && value >= 0 + ? value + : undefined; } -function modelMetadataFromRuntime(runtime: unknown): Record { - if (!isObject(runtime)) { - return {}; +function numericMetrics( + value: Record, +): Record { + const metrics: Record = {}; + for (const name of [ + "completion_tokens", + "prompt_cache_creation_tokens", + "prompt_cached_tokens", + "prompt_tokens", + "tokens", + ]) { + const metric = validMetric(value[name]); + if (metric !== undefined) { + metrics[name] = metric; + } } - const modelId = runtime["modelId"]; - return typeof modelId === "string" && !modelId.trim().startsWith("dynamic:") - ? modelMetadataFromModelId(modelId) - : {}; + return metrics; } -function modelMetadataFromModelId(modelId: string): Record { - const normalized = modelId.trim(); - if (!normalized) { - return {}; - } - - const slashIndex = normalized.indexOf("/"); - if (slashIndex > 0 && slashIndex < normalized.length - 1) { - return { - model: normalized.slice(slashIndex + 1), - provider: normalized.slice(0, slashIndex), - }; +function numericMetricContributions( + value: Record, +): Record> { + const contributions: Record> = {}; + for (const [idempotencyKey, contribution] of Object.entries(value)) { + if (isObject(contribution)) { + contributions[idempotencyKey] = numericMetrics(contribution); + } } + return contributions; +} +function modelMetadata( + modelId: string, + providerId: string, +): Record { + const provider = providerId.split(".")[0]?.trim(); + const model = modelId.trim(); return { - model: normalized, + ...(model.length === 0 ? {} : { model }), + ...(provider === undefined || provider.length === 0 ? {} : { provider }), }; } -function sessionIdFromContext(ctx: unknown): string | undefined { - if (!isObject(ctx)) { - return undefined; +function toError(error: unknown, fallback: string): Error { + if (error instanceof Error) { + return error; } - const session = ctx["session"]; - if (!isObject(session)) { - return undefined; + if (typeof error === "string" && error.length > 0) { + return new Error(error); } - const id = session["id"]; - return typeof id === "string" ? id : undefined; -} - -function toolMetadataFromTurn(turn: TurnState): Record { - const { model: _model, provider: _provider, ...metadata } = turn.metadata; - return metadata; -} - -function isToolCallAction( - action: unknown, -): action is EveRuntimeToolCallActionRequest { - return ( - isObject(action) && - action["kind"] === "tool-call" && - typeof action["callId"] === "string" && - typeof action["toolName"] === "string" && - isObject(action["input"]) - ); -} - -function isLocalSubagentCallAction( - action: unknown, -): action is Extract { - return ( - isObject(action) && - action["kind"] === "subagent-call" && - typeof action["callId"] === "string" && - isObject(action["input"]) - ); -} - -function isTraceableActionRequest( - action: unknown, -): action is - | EveRuntimeToolCallActionRequest - | Extract { - return isToolCallAction(action) || isLocalSubagentCallAction(action); -} - -function isToolResult( - result: unknown, -): result is EveRuntimeToolResultActionResult { - return ( - isObject(result) && - result["kind"] === "tool-result" && - typeof result["callId"] === "string" && - typeof result["toolName"] === "string" - ); -} - -function isSubagentResult( - result: unknown, -): result is Extract { - return ( - isObject(result) && - result["kind"] === "subagent-result" && - typeof result["callId"] === "string" && - typeof result["subagentName"] === "string" - ); -} - -function normalizedFinishReason( - finishReason: EveAssistantStepFinishReason, -): string { - switch (finishReason) { - case "content-filter": - return "content_filter"; - case "tool-calls": - return "tool_calls"; - default: - return finishReason; - } -} - -function errorFromMessage( - message: string, - code: string, - details?: unknown, -): Error { - const error = new Error(`${code}: ${message}`); - if (details !== undefined) { - error.cause = details; - } - return error; -} - -function actionResultError( - error: EveActionResultError | undefined, - output: unknown, -): Error { - if (error) { - return errorFromMessage(error.message, error.code); + if (isObject(error) && typeof error["message"] === "string") { + const code = typeof error["code"] === "string" ? error["code"] + ": " : ""; + const result = new Error(code + error["message"]); + result.cause = error; + return result; } - const result = new Error("Eve action failed"); - if (output !== undefined) { - result.cause = output; + const result = new Error(fallback); + if (error !== undefined) { + result.cause = error; } return result; } -function eventTime(event: { - readonly meta?: { readonly at: string }; -}): number | undefined { - if (!event.meta?.at) { - return undefined; - } - const timestamp = Date.parse(event.meta.at); - return Number.isFinite(timestamp) ? timestamp / 1000 : undefined; +function actionEndArgs( + acceptedAtMs: number | undefined, +): { endTime: number } | undefined { + return typeof acceptedAtMs === "number" && Number.isFinite(acceptedAtMs) + ? { endTime: acceptedAtMs / 1000 } + : undefined; } function turnKey(sessionId: string, turnId: string): string { - return `${sessionId}:${turnId}`; -} - -function toolKey(sessionId: string, callId: string): string { - return `${sessionId}:${callId}`; + return sessionId + ":" + turnId; } async function generateEveIds( - kind: EveEntityKind, + kind: "action" | "step" | "turn", ...parts: string[] ): Promise<{ rowId: string; spanId: string }> { const [rowId, spanId] = await Promise.all([ - deterministicEveId(`eve:row:${kind}`, ...parts), - deterministicEveId(`eve:${kind}`, ...parts), + deterministicEveId("eve:row:" + kind, ...parts), + deterministicEveId("eve:" + kind, ...parts), ]); return { rowId, spanId }; } async function deterministicEveId(...parts: string[]): Promise { const data = new TextEncoder().encode( - parts.map((part) => `${part.length}:${part}`).join("\0"), + parts.map((part) => part.length + ":" + part).join("\0"), ); const digest = await globalThis.crypto.subtle.digest("SHA-256", data); const bytes = Array.from(new Uint8Array(digest, 0, 16)); const hex = bytes.map((byte) => byte.toString(16).padStart(2, "0")).join(""); - return `${hex.slice(0, 8)}-${hex.slice(8, 12)}-${hex.slice(12, 16)}-${hex.slice(16, 20)}-${hex.slice(20)}`; + return ( + hex.slice(0, 8) + + "-" + + hex.slice(8, 12) + + "-" + + hex.slice(12, 16) + + "-" + + hex.slice(16, 20) + + "-" + + hex.slice(20) + ); } diff --git a/js/src/vendor-sdk-types/eve.ts b/js/src/vendor-sdk-types/eve.ts index 1910b3a87..5499fceb9 100644 --- a/js/src/vendor-sdk-types/eve.ts +++ b/js/src/vendor-sdk-types/eve.ts @@ -1,8 +1,8 @@ /** - * Vendored types for eve's authored hook APIs. + * Vendored types for eve's instrumentation provider API, introduced in 0.34.0. * * Keep this surface intentionally narrow. These types are not exported to SDK - * users and should only cover fields we read, correlate, or log. + * users and should only cover fields the Braintrust provider reads or returns. */ export type EveJsonValue = @@ -13,499 +13,197 @@ export type EveJsonValue = | EveJsonValue[] | { readonly [key: string]: EveJsonValue }; -export type EveJsonObject = { readonly [key: string]: EveJsonValue }; - -export interface EveHookContext { - readonly session: { - readonly id: string; - readonly parent?: { - readonly callId?: string; - readonly sessionId?: string; - readonly turn?: { - readonly id?: string; - }; - }; - }; +export interface EveInstrumentationState { + get(): EveJsonValue | undefined; + set(value: EveJsonValue | undefined): void; } -export type EveAssistantStepFinishReason = - | "content-filter" - | "error" - | "length" - | "other" - | "stop" - | "tool-calls"; +export interface EveInstrumentationHandlerContext { + readonly state: EveInstrumentationState; +} -export interface EveStreamEventMeta { - readonly at: string; +export interface EveInstrumentationSetupContext { + readonly agentName: string; + readonly environment?: "development" | "preview" | "production"; + readonly evaluation?: { readonly runId: string }; + readonly frameworkVersion?: string; } -export interface EveRuntimeToolCallActionRequest { - readonly callId: string; - readonly input: EveJsonObject; - readonly kind: "tool-call"; - readonly toolName: string; +export interface EveInstrumentationAttemptScope { + readonly attemptId: string; + readonly attemptIndex: number; + readonly rootSessionId?: string; + readonly sessionId: string; + readonly stepIndex: number; + readonly turnId: string; } -export interface EveRuntimeToolResultActionResult { +export interface EveInstrumentationParentLineage { readonly callId: string; - readonly isError?: boolean; - readonly kind: "tool-result"; - readonly output: EveJsonValue; - readonly toolName: string; + readonly sessionId: string; + readonly turnId: string; } -export type EveRuntimeActionRequest = - | EveRuntimeToolCallActionRequest - | { - readonly callId: string; - readonly input?: EveJsonObject; - readonly kind: "load-skill" | "remote-agent-call"; - readonly name?: string; - } - | { - readonly callId: string; - readonly input: EveJsonObject; - readonly kind: "subagent-call"; - readonly name?: string; - readonly subagentName?: string; - }; - -export type EveRuntimeActionResult = - | EveRuntimeToolResultActionResult - | { - readonly callId: string; - readonly isError?: boolean; - readonly kind: "load-skill-result"; - readonly output?: EveJsonValue; - readonly name?: string; - } - | { - readonly callId: string; - readonly isError?: boolean; - readonly kind: "subagent-result"; - readonly output?: EveJsonValue; - readonly subagentName?: string; - }; +export interface EveInstrumentationTraceContext { + readonly spanId: string; + readonly traceFlags: number; + readonly traceId: string; +} -export type EveActionResultStatus = "completed" | "failed" | "rejected"; +export interface EveInstrumentationUsage { + readonly inputTokenDetails?: { + readonly cacheReadTokens?: number; + readonly cacheWriteTokens?: number; + }; + readonly inputTokens?: number; + readonly outputTokens?: number; +} -export interface EveActionResultError { - readonly code: string; - readonly message: string; +export interface EveInstrumentationModelInput { + readonly instructions?: unknown; + readonly messages: readonly unknown[]; } -export type EveHandleMessageStreamEvent = - | { - readonly data: { - readonly invocation?: unknown; - readonly runtime?: { - readonly agentId: string; - readonly agentName?: string; - readonly eveVersion: string; - readonly modelId: string; - }; - }; - readonly meta?: EveStreamEventMeta; - readonly type: "session.started"; - } +export type EveInstrumentationContentPart = | { - readonly data: { - readonly sequence: number; - readonly turnId: string; - }; - readonly meta?: EveStreamEventMeta; - readonly type: "turn.started"; + readonly text: string; + readonly type: "text"; } | { - readonly data: { - readonly sequence: number; - readonly turnId: string; - }; - readonly meta?: EveStreamEventMeta; - readonly type: "turn.completed"; + readonly text: string; + readonly type: "reasoning"; } | { - readonly data: { - readonly message: string; - readonly sequence: number; - readonly turnId: string; - }; - readonly meta?: EveStreamEventMeta; - readonly type: "message.received"; - } - | { - readonly data: { - readonly finishReason: EveAssistantStepFinishReason; - readonly message: string | null; - readonly sequence: number; - readonly stepIndex: number; - readonly turnId: string; - }; - readonly meta?: EveStreamEventMeta; - readonly type: "message.completed"; - } - | { - readonly data: { - readonly reasoning: string; - readonly sequence: number; - readonly stepIndex: number; - readonly turnId: string; - }; - readonly meta?: EveStreamEventMeta; - readonly type: "reasoning.completed"; - } - | { - readonly data: { - readonly result: EveJsonValue; - readonly sequence: number; - readonly stepIndex: number; - readonly turnId: string; - }; - readonly meta?: EveStreamEventMeta; - readonly type: "result.completed"; - } - | { - readonly data: { - readonly sequence: number; - readonly stepIndex: number; - readonly turnId: string; - }; - readonly meta?: EveStreamEventMeta; - readonly type: "step.started"; - } - | { - readonly data: { - readonly finishReason: EveAssistantStepFinishReason; - readonly providerMetadata?: { - readonly gateway?: { - readonly generationId?: string; - }; - }; - readonly sequence: number; - readonly stepIndex: number; - readonly turnId: string; - readonly usage?: { - readonly cacheReadTokens?: number; - readonly cacheWriteTokens?: number; - readonly costUsd?: number; - readonly inputTokens?: number; - readonly outputTokens?: number; - }; - }; - readonly meta?: EveStreamEventMeta; - readonly type: "step.completed"; - } - | { - readonly data: { - readonly code: string; - readonly details?: EveJsonObject; - readonly message: string; - readonly sequence: number; - readonly stepIndex: number; - readonly turnId: string; - }; - readonly meta?: EveStreamEventMeta; - readonly type: "step.failed"; - } - | { - readonly data: { - readonly actions: readonly EveRuntimeActionRequest[]; - readonly sequence: number; - readonly stepIndex: number; - readonly turnId: string; - }; - readonly meta?: EveStreamEventMeta; - readonly type: "actions.requested"; - } - | { - readonly data: { - readonly error?: EveActionResultError; - readonly result: EveRuntimeActionResult; - readonly sequence: number; - readonly stepIndex: number; - readonly status: EveActionResultStatus; - readonly turnId: string; - }; - readonly meta?: EveStreamEventMeta; - readonly type: "action.result"; - } - | { - readonly data: { - readonly callId: string; - readonly childSessionId: string; - readonly name: string; - readonly remote?: { - readonly url?: string; - }; - readonly sequence: number; - readonly toolName?: string; - readonly turnId: string; - }; - readonly meta?: EveStreamEventMeta; - readonly type: "subagent.called"; - } - | { - readonly data: { - readonly callId: string; - readonly error?: EveActionResultError; - readonly output?: EveJsonValue; - readonly sequence: number; - readonly status?: EveActionResultStatus; - readonly subagentName: string; - readonly turnId: string; - }; - readonly meta?: EveStreamEventMeta; - readonly type: "subagent.completed"; - } - | { - readonly data: { - readonly code: string; - readonly details?: EveJsonObject; - readonly message: string; - readonly sequence: number; - readonly turnId: string; - }; - readonly meta?: EveStreamEventMeta; - readonly type: "turn.failed"; - } - | { - readonly data: { - readonly code: string; - readonly details?: EveJsonObject; - readonly message: string; - readonly sessionId: string; - }; - readonly meta?: EveStreamEventMeta; - readonly type: "session.failed"; + readonly callId: string; + readonly input: unknown; + readonly toolName: string; + readonly type: "tool-call"; } | { - readonly data: { - readonly wait: "next-user-message"; - }; - readonly meta?: EveStreamEventMeta; - readonly type: "session.waiting"; + readonly type: "tool-result"; } | { - readonly meta?: EveStreamEventMeta; - readonly type: "session.completed"; + readonly type: "tool-error"; }; -export interface EveHookDefinition { - readonly events?: { - readonly "*"?: ( - event: EveHandleMessageStreamEvent, - ctx: EveHookContext, - ) => void | Promise; - readonly [eventType: string]: - | (( - event: EveHandleMessageStreamEvent, - ctx: EveHookContext, - ) => void | Promise) - | undefined; - }; +export interface EveInstrumentationTurnStartedEvent { + readonly idempotencyKey: string; + readonly parentLineage?: EveInstrumentationParentLineage; + readonly parentTraceContext?: EveInstrumentationTraceContext; + readonly rootSessionId: string; + readonly sequence: number; + readonly sessionId: string; + readonly turnId: string; + readonly type: "turn.started"; } -export interface EveInstrumentationSetupContext { - readonly agentName: string; +export interface EveInstrumentationTurnSettledEvent { + readonly idempotencyKey: string; + readonly sessionId: string; + readonly turnId: string; + readonly type: "turn.cancelled" | "turn.completed"; } -type EveTextPart = { - readonly text: string; - readonly type: "text"; -}; - -type EveImagePart = { - readonly image: unknown; - readonly mediaType?: string; - readonly type: "image"; -}; - -type EveFilePart = { - readonly data: unknown; - readonly filename?: string; - readonly mediaType: string; - readonly type: "file"; -}; - -type EveReasoningPart = { - readonly text: string; - readonly type: "reasoning"; -}; - -type EveReasoningFilePart = { - readonly data: unknown; - readonly mediaType: string; - readonly type: "reasoning-file"; -}; - -type EveCustomPart = { - readonly kind: `${string}.${string}`; - readonly type: "custom"; -}; - -type EveToolCallPart = { - readonly input: unknown; - readonly providerExecuted?: boolean; - readonly toolCallId: string; - readonly toolName: string; - readonly type: "tool-call"; -}; - -type EveToolResultContentPart = - | EveTextPart - | EveFilePart - | { - readonly data: string; - readonly filename?: string; - readonly mediaType: string; - readonly type: "file-data"; - } - | { - readonly mediaType?: string; - readonly type: "file-url"; - readonly url: string; - } - | { - readonly fileId: string | Readonly>; - readonly type: "file-id" | "image-file-id"; - } - | { - readonly providerReference: Readonly>; - readonly type: "file-reference" | "image-file-reference"; - } - | { - readonly data: string; - readonly mediaType: string; - readonly type: "image-data"; - } - | { - readonly type: "image-url"; - readonly url: string; - } - | { readonly type: "custom" }; - -type EveToolResultOutput = - | { - readonly type: "text" | "error-text"; - readonly value: string; - } - | { - readonly type: "json" | "error-json"; - readonly value: EveJsonValue; - } - | { - readonly reason?: string; - readonly type: "execution-denied"; - } - | { - readonly type: "content"; - readonly value: readonly EveToolResultContentPart[]; - }; +export interface EveInstrumentationTurnFailedEvent { + readonly error?: unknown; + readonly idempotencyKey: string; + readonly sessionId: string; + readonly turnId: string; + readonly type: "turn.failed"; +} -type EveToolResultPart = { - readonly output: EveToolResultOutput; - readonly toolCallId: string; - readonly toolName: string; - readonly type: "tool-result"; -}; +export interface EveInstrumentationModelCallStartedEvent { + readonly idempotencyKey: string; + readonly input?: EveInstrumentationModelInput; + readonly model: { + readonly modelId: string; + readonly provider: string; + }; + readonly scope: EveInstrumentationAttemptScope; + readonly type: "model.call.started"; +} -type EveToolApprovalRequest = { - readonly approvalId: string; - readonly isAutomatic?: boolean; - readonly signature?: string; - readonly toolCallId: string; - readonly type: "tool-approval-request"; -}; +export interface EveInstrumentationModelCallCompletedEvent { + readonly content?: readonly EveInstrumentationContentPart[]; + readonly finishReason: string; + readonly idempotencyKey: string; + readonly scope: EveInstrumentationAttemptScope; + readonly type: "model.call.completed"; + readonly usage: EveInstrumentationUsage; +} -type EveToolApprovalResponse = { - readonly approvalId: string; - readonly approved: boolean; - readonly providerExecuted?: boolean; - readonly reason?: string; - readonly type: "tool-approval-response"; -}; +export interface EveInstrumentationModelCallFailedEvent { + readonly error?: unknown; + readonly idempotencyKey: string; + readonly scope: EveInstrumentationAttemptScope; + readonly type: "model.call.failed"; +} -export type EveModelMessageContentPart = - | EveTextPart - | EveImagePart - | EveFilePart - | EveReasoningPart - | EveReasoningFilePart - | EveCustomPart - | EveToolCallPart - | EveToolResultPart - | EveToolApprovalRequest - | EveToolApprovalResponse - | EveToolResultContentPart; +export interface EveInstrumentationStepAttemptCompletedEvent { + readonly idempotencyKey: string; + readonly scope: EveInstrumentationAttemptScope; + readonly type: "step.attempt.completed"; +} -export type EveSystemModelMessage = { - readonly content: string; - readonly role: "system"; -}; +export interface EveInstrumentationStepAttemptFailedEvent { + readonly error?: unknown; + readonly idempotencyKey: string; + readonly scope: EveInstrumentationAttemptScope; + readonly type: "step.attempt.failed"; +} -export type EveModelMessage = - | EveSystemModelMessage - | { - readonly content: - | string - | readonly (EveTextPart | EveImagePart | EveFilePart)[]; - readonly role: "user"; - } - | { - readonly content: - | string - | readonly ( - | EveTextPart - | EveCustomPart - | EveFilePart - | EveReasoningPart - | EveReasoningFilePart - | EveToolCallPart - | EveToolResultPart - | EveToolApprovalRequest - )[]; - readonly role: "assistant"; - } - | { - readonly content: readonly ( - | EveToolResultPart - | EveToolApprovalResponse - )[]; - readonly role: "tool"; - }; +export interface EveInstrumentationActionStartedEvent { + readonly callId: string; + readonly idempotencyKey: string; + readonly input?: unknown; + readonly name: string; + readonly scope: EveInstrumentationAttemptScope; + readonly type: "action.started"; +} -export interface EveInstrumentationModelInput { - readonly instructions?: string | readonly EveSystemModelMessage[]; - readonly messages: readonly EveModelMessage[]; +export interface EveInstrumentationActionCompletedEvent { + readonly acceptedAtMs?: number; + readonly idempotencyKey: string; + readonly output: + | { readonly output?: unknown; readonly type: "result" } + | { readonly error?: unknown; readonly type: "error" }; + readonly scope: EveInstrumentationAttemptScope; + readonly type: "action.completed"; } -export interface EveInstrumentationStepStartedEventInput { - readonly modelInput: EveInstrumentationModelInput; - readonly session: { - readonly id: string; - }; - readonly step: { - readonly index: number; - }; - readonly turn: { - readonly id: string; - readonly sequence: number; - }; +export interface EveInstrumentationActionFailedEvent { + readonly acceptedAtMs?: number; + readonly error?: unknown; + readonly errorCode?: string; + readonly idempotencyKey: string; + readonly outcome: "abandoned" | "cancelled" | "failed" | "rejected"; + readonly scope: EveInstrumentationAttemptScope; + readonly type: "action.failed"; } +type EveInstrumentationHandler = ( + event: TEvent, + context: EveInstrumentationHandlerContext, +) => void | PromiseLike; + export interface EveInstrumentationDefinition { - readonly events?: { - readonly "step.started"?: ( - input: EveInstrumentationStepStartedEventInput, - ) => void | { readonly runtimeContext?: EveJsonObject }; - readonly [eventType: string]: - | (( - input: EveInstrumentationStepStartedEventInput, - ) => void | { readonly runtimeContext?: EveJsonObject }) - | undefined; + readonly capture: "content"; + readonly events: { + readonly "action.completed": EveInstrumentationHandler; + readonly "action.failed": EveInstrumentationHandler; + readonly "action.started": EveInstrumentationHandler; + readonly "model.call.completed": EveInstrumentationHandler; + readonly "model.call.failed": EveInstrumentationHandler; + readonly "model.call.started": EveInstrumentationHandler; + readonly "step.attempt.completed": EveInstrumentationHandler; + readonly "step.attempt.failed": EveInstrumentationHandler; + readonly "turn.cancelled": EveInstrumentationHandler; + readonly "turn.completed": EveInstrumentationHandler; + readonly "turn.failed": EveInstrumentationHandler; + readonly "turn.started": EveInstrumentationHandler; }; - readonly recordInputs?: boolean; - readonly recordOutputs?: boolean; - readonly setup?: (context: EveInstrumentationSetupContext) => void; + readonly flush: () => Promise; + readonly setup?: ( + context: EveInstrumentationSetupContext, + ) => void | PromiseLike; } diff --git a/js/src/wrappers/attachment-utils.ts b/js/src/wrappers/attachment-utils.ts index 7fb2b96f5..df7544fce 100644 --- a/js/src/wrappers/attachment-utils.ts +++ b/js/src/wrappers/attachment-utils.ts @@ -242,6 +242,26 @@ export function processInputAttachments(input: any): any { } } + // AI SDK tool output content format + if ( + (node.type === "image-data" || node.type === "file-data") && + node.data + ) { + const mediaType = node.mediaType || "application/octet-stream"; + const filename = + node.filename || + `output_${node.type === "image-data" ? "image" : "file"}_${attachmentIndex}.${getExtensionFromMediaType(mediaType)}`; + const attachment = toAttachment(node.data, mediaType, filename); + + if (attachment) { + attachmentIndex++; + return { + ...node, + data: attachment, + }; + } + } + const processed: Record = {}; for (const [key, value] of Object.entries(node)) { processed[key] = processNode(value); From 4cd71d131a9f71e75698fade337294dfbf4df58d Mon Sep 17 00:00:00 2001 From: Luca Forstner Date: Wed, 26 Aug 2026 20:08:53 +0200 Subject: [PATCH 5/8] ref!: Remove Zod derived types from public SDK declarations (#2404) ref: [SDK-226](https://linear.app/braintrustdata/issue/SDK-226) Replace Zod-inferred public types with explicit TypeScript types to reduce declaration size and downstream compile cost. --- .changeset/remove-zod-derived-types.md | 5 + AGENTS.md | 15 +++ js/src/eval-parameters.ts | 35 +++--- js/src/exports.ts | 7 +- js/src/framework-types.ts | 2 +- js/src/framework.ts | 14 +-- js/src/framework2.ts | 24 ++-- js/src/functions/invoke.ts | 14 +-- js/src/functions/stream.ts | 91 ++++++++------ js/src/gitutil.ts | 8 +- js/src/graph-framework.ts | 14 +-- js/src/isomorph.ts | 8 +- js/src/logger.ts | 114 ++++++++++++------ js/src/prompt-schemas.ts | 59 ++++++--- js/src/public-types.test.ts | 69 +++++++++++ js/src/sandbox.ts | 2 +- .../api-compatibility.test.ts | 50 ++++++++ js/util/git_fields.ts | 2 +- js/util/object.ts | 6 +- js/util/span_identifier_v3.ts | 35 +++++- js/util/span_identifier_v4.ts | 9 +- 21 files changed, 416 insertions(+), 167 deletions(-) create mode 100644 .changeset/remove-zod-derived-types.md create mode 100644 js/src/public-types.test.ts diff --git a/.changeset/remove-zod-derived-types.md b/.changeset/remove-zod-derived-types.md new file mode 100644 index 000000000..c741bede7 --- /dev/null +++ b/.changeset/remove-zod-derived-types.md @@ -0,0 +1,5 @@ +--- +"braintrust": major +--- + +ref!: Remove Zod derived types from public SDK declarations diff --git a/AGENTS.md b/AGENTS.md index 6338e8830..1b62e3529 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -25,6 +25,21 @@ mise install # Install toolchain and dependencies pnpm run build # Build all workspace packages (from repo root) ``` +## Public TypeScript APIs + +Do not derive SDK-owned public TypeScript types from Zod schemas (for example, +with `z.infer`, `z.input`, `z.output`, or equivalent schema-derived aliases). +Define public API types explicitly with interfaces, type aliases, or generated +plain types. Generic APIs may still infer types from caller-provided schemas. +When exporting a runtime validator, give it a compact public type such as +`z.ZodType` and test that the validator and public type stay in sync. + +Zod-derived public declarations can expand into large schema implementation +graphs. Those declarations are expensive for downstream TypeScript consumers to +parse, instantiate, and type-check, increasing compile time, declaration size, +and memory usage. They also expose validation-library implementation details as +part of the SDK's API surface. + ## Instrumentation Use the normal Orchestrion config plus plugin/channel path by default. Special-case source patches should be rare exceptions only when the target SDK cannot be instrumented through the standard transformer path, and the reason should be documented next to the patch. diff --git a/js/src/eval-parameters.ts b/js/src/eval-parameters.ts index 1154fcfb8..bc15dc105 100644 --- a/js/src/eval-parameters.ts +++ b/js/src/eval-parameters.ts @@ -2,30 +2,25 @@ import { z } from "zod/v3"; import Ajv from "ajv"; import { Prompt, RemoteEvalParameters } from "./logger"; import { - promptDefinitionWithToolsSchema, promptDefinitionToPromptData, + type PromptDefinitionWithTools, } from "./prompt-schemas"; import { PromptData as promptDataSchema } from "./generated_types"; -// Schema for evaluation parameters -export const evalParametersSchema = z.record( - z.string(), - z.union([ - z.object({ - type: z.literal("prompt"), - default: promptDefinitionWithToolsSchema.optional(), - description: z.string().optional(), - }), - z.object({ - type: z.literal("model"), - default: z.string().optional(), - description: z.string().optional(), - }), - z.instanceof(z.ZodType), // For Zod schemas - ]), -); - -export type EvalParameters = z.infer; +export type EvalParameters = Record< + string, + | { + type: "prompt"; + default?: PromptDefinitionWithTools; + description?: string; + } + | { + type: "model"; + default?: string; + description?: string; + } + | z.ZodTypeAny +>; // Type helper to infer the type of a parameter value type InferParameterValue = T extends { type: "prompt" } diff --git a/js/src/exports.ts b/js/src/exports.ts index 9ed496ceb..e8286a959 100644 --- a/js/src/exports.ts +++ b/js/src/exports.ts @@ -1,3 +1,7 @@ +import type { z } from "zod/v3"; +import { AttachmentReference as attachmentReferenceSchema } from "./generated_types"; +import type { AttachmentReferenceType } from "./generated_plain_types"; + export type { AnyDataset, AttachmentParams, @@ -322,7 +326,8 @@ export type { export { addAzureBlobHeaders, LazyValue } from "./util"; -export { AttachmentReference } from "./generated_types"; +export const AttachmentReference: z.ZodType = + attachmentReferenceSchema; export type { EvalParameters } from "./eval-parameters"; diff --git a/js/src/framework-types.ts b/js/src/framework-types.ts index b5183cc7d..0f9d45c26 100644 --- a/js/src/framework-types.ts +++ b/js/src/framework-types.ts @@ -1,4 +1,4 @@ -import { type IfExistsType as IfExists } from "./generated_types"; +import type { IfExistsType as IfExists } from "./generated_plain_types"; export type GenericFunction = | ((input: Input) => Output) diff --git a/js/src/framework.ts b/js/src/framework.ts index 9a463533b..540b95dc6 100644 --- a/js/src/framework.ts +++ b/js/src/framework.ts @@ -7,13 +7,13 @@ import { SpanTypeAttribute, spanObjectTypeV3ToTypedString, } from "../util/index"; -import { - type GitMetadataSettingsType as GitMetadataSettings, - ObjectReference as ObjectReferenceSchema, - type ObjectReferenceType as ObjectReference, - type RepoInfoType as RepoInfo, - type SSEProgressEventDataType as SSEProgressEventData, -} from "./generated_types"; +import { ObjectReference as ObjectReferenceSchema } from "./generated_types"; +import type { + GitMetadataSettingsType as GitMetadataSettings, + ObjectReferenceType as ObjectReference, + RepoInfoType as RepoInfo, + SSEProgressEventDataType as SSEProgressEventData, +} from "./generated_plain_types"; import { queue } from "async"; import iso from "./isomorph"; diff --git a/js/src/framework2.ts b/js/src/framework2.ts index f9220b6e1..ab4ee52c5 100644 --- a/js/src/framework2.ts +++ b/js/src/framework2.ts @@ -3,17 +3,17 @@ import type { Trace } from "./trace"; import iso from "./isomorph"; import { slugify } from "../util/string_util"; import { z } from "zod/v3"; -import { - type FunctionTypeEnumType as FunctionType, - type IfExistsType as IfExists, - type SavedFunctionIdType as SavedFunctionId, - type PromptBlockDataType as PromptBlockData, - type PromptDataType as PromptData, - type ToolFunctionDefinitionType as ToolFunctionDefinition, - FunctionData as functionDataSchema, - Project as projectSchema, - type ExtendedSavedFunctionIdType as ExtendedSavedFunctionId, -} from "./generated_types"; +import { Project as projectSchema } from "./generated_types"; +import type { + FunctionTypeEnumType as FunctionType, + IfExistsType as IfExists, + SavedFunctionIdType as SavedFunctionId, + PromptBlockDataType as PromptBlockData, + PromptDataType as PromptData, + ToolFunctionDefinitionType as ToolFunctionDefinition, + ExtendedSavedFunctionIdType as ExtendedSavedFunctionId, + FunctionDataType, +} from "./generated_plain_types"; import { loadPrettyXact, TransactionId } from "../util/index"; import { _internalGetGlobalState, @@ -782,7 +782,7 @@ interface FunctionEvent { name: string; description: string; prompt_data?: PromptData; - function_data: z.infer; + function_data: FunctionDataType; function_type?: FunctionType; if_exists?: IfExists; tags?: string[]; diff --git a/js/src/functions/invoke.ts b/js/src/functions/invoke.ts index f7bbb35f5..82fd40de4 100644 --- a/js/src/functions/invoke.ts +++ b/js/src/functions/invoke.ts @@ -1,10 +1,10 @@ -import { - FunctionId as functionIdSchema, - type InvokeFunctionType as InvokeFunctionRequest, - type ChatCompletionMessageParamType as Message, - type StreamingModeType as StreamingMode, - type FunctionTypeEnumType as FunctionType, -} from "../generated_types"; +import { FunctionId as functionIdSchema } from "../generated_types"; +import type { + InvokeFunctionType as InvokeFunctionRequest, + ChatCompletionMessageParamType as Message, + StreamingModeType as StreamingMode, + FunctionTypeEnumType as FunctionType, +} from "../generated_plain_types"; import { _internalGetGlobalState, BraintrustState, diff --git a/js/src/functions/stream.ts b/js/src/functions/stream.ts index 300895075..392e6b337 100644 --- a/js/src/functions/stream.ts +++ b/js/src/functions/stream.ts @@ -1,9 +1,13 @@ import { - type CallEventType as CallEventSchema, CallEvent as callEventSchema, SSEConsoleEventData as sseConsoleEventDataSchema, SSEProgressEventData as sseProgressEventDataSchema, } from "../generated_types"; +import type { + CallEventType as CallEvent, + SSEConsoleEventDataType, + SSEProgressEventDataType, +} from "../generated_plain_types"; import { createParser, EventSourceParser, @@ -12,46 +16,55 @@ import { } from "eventsource-parser"; import { z } from "zod/v3"; -export const braintrustStreamChunkSchema = z.union([ - z.object({ - type: z.literal("text_delta"), - data: z.string(), - }), - z.object({ - type: z.literal("reasoning_delta"), - data: z.string(), - }), - z.object({ - type: z.literal("json_delta"), - data: z.string(), - }), - z.object({ - type: z.literal("error"), - data: z.string(), - }), - z.object({ - type: z.literal("console"), - data: sseConsoleEventDataSchema, - }), - z.object({ - type: z.literal("progress"), - data: sseProgressEventDataSchema, - }), - z.object({ - type: z.literal("start"), - data: z.string(), - }), - z.object({ - type: z.literal("done"), - data: z.string(), - }), -]); - /** * A chunk of data from a Braintrust stream. Each chunk type matches * an SSE event type. */ -export type BraintrustStreamChunk = z.infer; +export type BraintrustStreamChunk = + | { type: "text_delta"; data: string } + | { type: "reasoning_delta"; data: string } + | { type: "json_delta"; data: string } + | { type: "error"; data: string } + | { type: "console"; data: SSEConsoleEventDataType } + | { type: "progress"; data: SSEProgressEventDataType } + | { type: "start"; data: string } + | { type: "done"; data: string }; + +export const braintrustStreamChunkSchema: z.ZodType = + z.union([ + z.object({ + type: z.literal("text_delta"), + data: z.string(), + }), + z.object({ + type: z.literal("reasoning_delta"), + data: z.string(), + }), + z.object({ + type: z.literal("json_delta"), + data: z.string(), + }), + z.object({ + type: z.literal("error"), + data: z.string(), + }), + z.object({ + type: z.literal("console"), + data: sseConsoleEventDataSchema, + }), + z.object({ + type: z.literal("progress"), + data: sseProgressEventDataSchema, + }), + z.object({ + type: z.literal("start"), + data: z.string(), + }), + z.object({ + type: z.literal("done"), + data: z.string(), + }), + ]); /** * A Braintrust stream. This is a wrapper around a ReadableStream of `BraintrustStreamChunk`, @@ -163,7 +176,7 @@ export class BraintrustStream { return this.memoizedFinalValue; } - static parseRawEvent(event: CallEventSchema): BraintrustStreamChunk { + static parseRawEvent(event: CallEvent): BraintrustStreamChunk { switch (event.event) { case "text_delta": return { @@ -212,7 +225,7 @@ export class BraintrustStream { } } - static serializeRawEvent(event: BraintrustStreamChunk): CallEventSchema { + static serializeRawEvent(event: BraintrustStreamChunk): CallEvent { switch (event.type) { case "text_delta": return { diff --git a/js/src/gitutil.ts b/js/src/gitutil.ts index 8e318ac98..9a687ebb3 100644 --- a/js/src/gitutil.ts +++ b/js/src/gitutil.ts @@ -1,7 +1,7 @@ -import { - type GitMetadataSettingsType as GitMetadataSettings, - type RepoInfoType as RepoInfo, -} from "./generated_types"; +import type { + GitMetadataSettingsType as GitMetadataSettings, + RepoInfoType as RepoInfo, +} from "./generated_plain_types"; import { debugLogger } from "./debug-logger"; import { runGitCommand } from "./git-command"; diff --git a/js/src/graph-framework.ts b/js/src/graph-framework.ts index f60ff3fba..5d6766da6 100644 --- a/js/src/graph-framework.ts +++ b/js/src/graph-framework.ts @@ -1,11 +1,11 @@ import { newId, Prompt } from "./logger"; -import { - type FunctionIdType as FunctionId, - type GraphDataType as GraphData, - type GraphNodeType as GraphNode, - type GraphEdgeType as GraphEdge, - type PromptBlockDataType as PromptBlockData, -} from "./generated_types"; +import type { + FunctionIdType as FunctionId, + GraphDataType as GraphData, + GraphNodeType as GraphNode, + GraphEdgeType as GraphEdge, + PromptBlockDataType as PromptBlockData, +} from "./generated_plain_types"; export interface BuildContext { getFunctionId(functionObj: unknown): Promise; diff --git a/js/src/isomorph.ts b/js/src/isomorph.ts index d23ddcc7e..0b500bcec 100644 --- a/js/src/isomorph.ts +++ b/js/src/isomorph.ts @@ -1,7 +1,7 @@ -import { - type GitMetadataSettingsType as GitMetadataSettings, - type RepoInfoType as RepoInfo, -} from "./generated_types"; +import type { + GitMetadataSettingsType as GitMetadataSettings, + RepoInfoType as RepoInfo, +} from "./generated_plain_types"; import { newGlobalTracingChannel, type GlobalHookAsyncLocalStorage, diff --git a/js/src/logger.ts b/js/src/logger.ts index 9733c3030..fcf88d9a4 100644 --- a/js/src/logger.ts +++ b/js/src/logger.ts @@ -66,37 +66,38 @@ import { getObjValueByPath, } from "./util"; import { - type AnyModelParamsType as AnyModelParam, AttachmentReference as attachmentReferenceSchema, - type AttachmentReferenceType as AttachmentReference, BraintrustAttachmentReference as BraintrustAttachmentReferenceSchema, - type BraintrustAttachmentReferenceType as BraintrustAttachmentReference, BraintrustModelParams as braintrustModelParamsSchema, ChatCompletionTool as chatCompletionToolSchema, - type ChatCompletionToolType as ChatCompletionTool, ExternalAttachmentReference as ExternalAttachmentReferenceSchema, - type ExternalAttachmentReferenceType as ExternalAttachmentReference, - type ModelParamsType as ModelParams, ResponseFormatJsonSchema as responseFormatJsonSchemaSchema, AttachmentStatus as attachmentStatusSchema, - type AttachmentStatusType as AttachmentStatus, GitMetadataSettings as gitMetadataSettingsSchema, - type GitMetadataSettingsType as GitMetadataSettings, - type ChatCompletionMessageParamType as Message, - type ChatCompletionOpenAIMessageParamType as OpenAIMessage, DatasetSnapshot as datasetSnapshotSchema, - type DatasetSnapshotType as DatasetSnapshot, PromptData as promptDataSchema, - type PromptDataType as PromptData, Prompt as promptSchema, - type PromptType as PromptRow, - type PromptSessionEventType as PromptSessionEvent, - type RepoInfoType as RepoInfo, - type ObjectReferenceType as ObjectReference, - type PromptBlockDataType as PromptBlockData, - type ResponseFormatJsonSchemaType as ResponseFormatJsonSchema, - type ObjectReferenceType, } from "./generated_types"; +import type { + AnyModelParamsType as AnyModelParam, + AttachmentReferenceType as AttachmentReference, + BraintrustAttachmentReferenceType as BraintrustAttachmentReference, + ChatCompletionToolType as ChatCompletionTool, + ExternalAttachmentReferenceType as ExternalAttachmentReference, + ModelParamsType as ModelParams, + AttachmentStatusType as AttachmentStatus, + GitMetadataSettingsType as GitMetadataSettings, + ChatCompletionMessageParamType as Message, + ChatCompletionOpenAIMessageParamType as OpenAIMessage, + DatasetSnapshotType as DatasetSnapshot, + PromptDataType as PromptData, + PromptType as PromptRow, + PromptSessionEventType as PromptSessionEvent, + RepoInfoType as RepoInfo, + ObjectReferenceType as ObjectReference, + PromptBlockDataType as PromptBlockData, + ResponseFormatJsonSchemaType as ResponseFormatJsonSchema, +} from "./generated_plain_types"; const BRAINTRUST_ATTACHMENT = BraintrustAttachmentReferenceSchema.shape.type.value; @@ -120,16 +121,21 @@ const datasetRestorePreviewResultSchema = z.object({ rows_to_restore: z.number(), rows_to_delete: z.number(), }); -export type DatasetRestorePreviewResult = z.infer< - typeof datasetRestorePreviewResultSchema ->; +export type DatasetRestorePreviewResult = { + rows_to_restore: number; + rows_to_delete: number; +}; const datasetRestoreResultSchema = z.object({ xact_id: z.string().nullable(), rows_restored: z.number(), rows_deleted: z.number(), }); -export type DatasetRestoreResult = z.infer; +export type DatasetRestoreResult = { + xact_id: string | null; + rows_restored: number; + rows_deleted: number; +}; const parametersRowSchema = z.object({ id: z.string().uuid(), @@ -148,7 +154,21 @@ const parametersRowSchema = z.object({ .union([z.object({}).partial().passthrough(), z.null()]) .optional(), }); -type ParametersRow = z.infer; +type ParametersRow = { + id: string; + _xact_id: string; + project_id: string; + name: string; + slug: string; + description?: string | null; + function_type: "parameters"; + function_data: { + type: "parameters"; + data?: Record; + __schema: Record; + }; + metadata?: Record | null; +}; import { waitUntil } from "@vercel/functions"; import { @@ -714,7 +734,18 @@ const loginSchema = z.strictObject({ debugLogLevelDisabled: z.boolean().optional(), }); -export type SerializedBraintrustState = z.infer; +export type SerializedBraintrustState = { + appUrl: string; + appPublicUrl: string; + orgName: string; + apiUrl: string; + proxyUrl: string; + loginToken: string; + orgId?: string | null; + gitMetadataSettings?: GitMetadataSettings | null; + debugLogLevel?: "error" | "warn" | "info" | "debug"; + debugLogLevelDisabled?: boolean; +}; let stateNonce = 0; @@ -1863,7 +1894,10 @@ const attachmentMetadataSchema = z.object({ status: attachmentStatusSchema, }); -type AttachmentMetadata = z.infer; +type AttachmentMetadata = { + downloadUrl: string; + status: AttachmentStatus; +}; /** * A readonly alternative to `Attachment`, which can be used for fetching @@ -2784,14 +2818,22 @@ function castLogger( return logger as unknown as Logger; } -export const logs3OverflowUploadSchema = z.object({ - method: z.enum(["PUT", "POST"]), - signedUrl: z.string().url(), - headers: z.record(z.string()).optional(), - fields: z.record(z.string()).optional(), - key: z.string().min(1), -}); -export type Logs3OverflowUpload = z.infer; +export type Logs3OverflowUpload = { + method: "PUT" | "POST"; + signedUrl: string; + headers?: Record; + fields?: Record; + key: string; +}; + +export const logs3OverflowUploadSchema: z.ZodType = + z.object({ + method: z.enum(["PUT", "POST"]), + signedUrl: z.string().url(), + headers: z.record(z.string()).optional(), + fields: z.record(z.string()).optional(), + key: z.string().min(1), + }); export type Logs3OverflowInputRow = { object_ids: Record; @@ -8375,7 +8417,7 @@ export class Dataset< metadata?: Record; tags?: string[]; output?: unknown; - origin?: ObjectReferenceType; + origin?: ObjectReference; isMerge?: boolean; }): LazyValue { return new LazyValue(async () => { @@ -8434,7 +8476,7 @@ export class Dataset< readonly metadata?: Record; readonly id?: string; readonly output?: unknown; - readonly origin?: ObjectReferenceType; + readonly origin?: ObjectReference; }): string { this.validateEvent({ metadata, expected, output, tags }); diff --git a/js/src/prompt-schemas.ts b/js/src/prompt-schemas.ts index c4203f90e..46ad5f307 100644 --- a/js/src/prompt-schemas.ts +++ b/js/src/prompt-schemas.ts @@ -1,15 +1,23 @@ import { z } from "zod/v3"; import { ToolFunctionDefinition as toolFunctionDefinitionSchema, - type ToolFunctionDefinitionType as ToolFunctionDefinition, ChatCompletionMessageParam as chatCompletionMessageParamSchema, ModelParams as modelParamsSchema, - type PromptBlockDataType as PromptBlockData, - type PromptDataType as PromptData, } from "./generated_types"; +import type { + ToolFunctionDefinitionType as ToolFunctionDefinition, + ChatCompletionMessageParamType, + ModelParamsType, + PromptBlockDataType as PromptBlockData, + PromptDataType as PromptData, +} from "./generated_plain_types"; // This roughly maps to promptBlockDataSchema, but is more ergonomic for the user. -export const promptContentsSchema = z.union([ +export type PromptContents = + | { prompt: string } + | { messages: ChatCompletionMessageParamType[] }; + +const internalPromptContentsSchema = z.union([ z.object({ prompt: z.string(), }), @@ -17,10 +25,20 @@ export const promptContentsSchema = z.union([ messages: z.array(chatCompletionMessageParamSchema), }), ]); +export const promptContentsSchema: z.ZodType< + PromptContents, + z.ZodTypeDef, + unknown +> = internalPromptContentsSchema; -export type PromptContents = z.infer; +export type PromptDefinition = PromptContents & { + model: string; + params?: ModelParamsType; + templateFormat?: "mustache" | "nunjucks" | "none"; + environments?: string[]; +}; -export const promptDefinitionSchema = promptContentsSchema.and( +const internalPromptDefinitionSchema = internalPromptContentsSchema.and( z.object({ model: z.string(), params: modelParamsSchema.optional(), @@ -28,18 +46,27 @@ export const promptDefinitionSchema = promptContentsSchema.and( environments: z.array(z.string()).optional(), }), ); +export const promptDefinitionSchema: z.ZodType< + PromptDefinition, + z.ZodTypeDef, + unknown +> = internalPromptDefinitionSchema; -export type PromptDefinition = z.infer; - -export const promptDefinitionWithToolsSchema = promptDefinitionSchema.and( - z.object({ - tools: z.array(toolFunctionDefinitionSchema).optional(), - }), -); +export type PromptDefinitionWithTools = PromptDefinition & { + tools?: ToolFunctionDefinition[]; +}; -export type PromptDefinitionWithTools = z.infer< - typeof promptDefinitionWithToolsSchema ->; +const internalPromptDefinitionWithToolsSchema = + internalPromptDefinitionSchema.and( + z.object({ + tools: z.array(toolFunctionDefinitionSchema).optional(), + }), + ); +export const promptDefinitionWithToolsSchema: z.ZodType< + PromptDefinitionWithTools, + z.ZodTypeDef, + unknown +> = internalPromptDefinitionWithToolsSchema; export function promptDefinitionToPromptData( promptDefinition: PromptDefinition, diff --git a/js/src/public-types.test.ts b/js/src/public-types.test.ts new file mode 100644 index 000000000..0583994df --- /dev/null +++ b/js/src/public-types.test.ts @@ -0,0 +1,69 @@ +import { expectTypeOf, test } from "vitest"; +import { z } from "zod/v3"; + +import { + AttachmentReference, + braintrustStreamChunkSchema, + logs3OverflowUploadSchema, + promptContentsSchema, + promptDefinitionSchema, + promptDefinitionWithToolsSchema, + type BraintrustStreamChunk, + type EvalParameters, + type Logs3OverflowUpload, + type PromptContents, + type PromptDefinition, + type PromptDefinitionWithTools, +} from "./exports"; +import type { InferParameters } from "./eval-parameters"; +import type { AttachmentReferenceType } from "./generated_plain_types"; +import type { Prompt } from "./logger"; +import { + spanComponentsV3Schema, + type SpanComponentsV3Data, +} from "../util/span_identifier_v3"; +import { + spanComponentsV4Schema, + type SpanComponentsV4Data, +} from "../util/span_identifier_v4"; + +test("exported validators preserve their public output types", () => { + expectTypeOf< + z.infer + >().toEqualTypeOf(); + expectTypeOf< + z.infer + >().toEqualTypeOf(); + expectTypeOf< + z.infer + >().toEqualTypeOf(); + expectTypeOf< + z.infer + >().toEqualTypeOf(); + expectTypeOf< + z.infer + >().toEqualTypeOf(); + expectTypeOf< + z.infer + >().toEqualTypeOf(); + expectTypeOf< + z.infer + >().toEqualTypeOf(); + expectTypeOf< + z.infer + >().toEqualTypeOf(); +}); + +test("evaluation parameters retain custom schema inference", () => { + const parameters = { + subject: z.string(), + model: { type: "model" as const }, + prompt: { type: "prompt" as const }, + } satisfies EvalParameters; + + expectTypeOf>().toEqualTypeOf<{ + subject: string; + model: string; + prompt: Prompt; + }>(); +}); diff --git a/js/src/sandbox.ts b/js/src/sandbox.ts index 8517c801d..cfafa8c98 100644 --- a/js/src/sandbox.ts +++ b/js/src/sandbox.ts @@ -1,6 +1,6 @@ import { z } from "zod/v3"; import { slugify } from "../util/string_util"; -import { type IfExistsType } from "./generated_types"; +import type { IfExistsType } from "./generated_plain_types"; import { type BraintrustState, _internalGetGlobalState } from "./logger"; /** diff --git a/js/tests/api-compatibility/api-compatibility.test.ts b/js/tests/api-compatibility/api-compatibility.test.ts index f11e3d847..2fd785357 100644 --- a/js/tests/api-compatibility/api-compatibility.test.ts +++ b/js/tests/api-compatibility/api-compatibility.test.ts @@ -2948,6 +2948,56 @@ describe("API Compatibility", () => { expect(fs.existsSync(path.join(tempDir, "package"))).toBe(true); }); + test("keeps public declarations free of expanded Zod schema graphs", () => { + const declarationRoot = path.join(__dirname, "..", ".."); + const publicDeclarationPaths = [ + "dist/index.d.ts", + "dist/browser.d.ts", + "util/dist/index.d.ts", + ]; + + for (const declarationTypesPath of publicDeclarationPaths) { + const declarationPath = path.join(declarationRoot, declarationTypesPath); + const declaration = fs.readFileSync(declarationPath, "utf8"); + + expect(declaration).not.toMatch(/z\.infer { if (!publishedVersion) { console.log("Skipping test: No published version available"); diff --git a/js/util/git_fields.ts b/js/util/git_fields.ts index 8eb06d625..482a43526 100644 --- a/js/util/git_fields.ts +++ b/js/util/git_fields.ts @@ -1,4 +1,4 @@ -import { GitMetadataSettingsType as GitMetadataSettings } from "./generated_types"; +import type { GitMetadataSettingsType as GitMetadataSettings } from "./generated_plain_types"; export function mergeGitMetadataSettings( s1: GitMetadataSettings, diff --git a/js/util/object.ts b/js/util/object.ts index 7ea593005..941c6264a 100644 --- a/js/util/object.ts +++ b/js/util/object.ts @@ -1,7 +1,7 @@ -import { +import type { AsyncScoringControlType as AsyncScoringControl, - type ObjectReferenceType, -} from "./generated_types"; + ObjectReferenceType, +} from "./generated_plain_types"; import { Source, ASYNC_SCORING_CONTROL_FIELD, diff --git a/js/util/span_identifier_v3.ts b/js/util/span_identifier_v3.ts index f057f798b..78a753d53 100644 --- a/js/util/span_identifier_v3.ts +++ b/js/util/span_identifier_v3.ts @@ -38,7 +38,8 @@ export enum SpanObjectTypeV3 { PLAYGROUND_LOGS = 3, } -export const spanObjectTypeV3EnumSchema = z.nativeEnum(SpanObjectTypeV3); +export const spanObjectTypeV3EnumSchema: z.ZodType = + z.nativeEnum(SpanObjectTypeV3); export function spanObjectTypeV3ToTypedString( objectType: SpanObjectTypeV3, @@ -81,7 +82,35 @@ const _INTERNAL_SPAN_COMPONENT_UUID_FIELDS_ID_TO_NAME: Record< [InternalSpanComponentUUIDFields.ROOT_SPAN_ID]: "root_span_id", }; -export const spanComponentsV3Schema = z +type SpanObjectMetadata = + | { + object_id?: string | null; + compute_object_metadata_args?: null; + } + | { + object_id?: null; + compute_object_metadata_args: Record; + }; + +type SpanRowIds = + | { + row_id: string; + span_id: string; + root_span_id: string; + } + | { + row_id?: null; + span_id?: null; + root_span_id?: null; + }; + +export type SpanComponentsV3Data = { + object_type: SpanObjectTypeV3; + propagated_event?: Record | null; +} & SpanObjectMetadata & + SpanRowIds; + +export const spanComponentsV3Schema: z.ZodType = z .object({ object_type: spanObjectTypeV3EnumSchema, // TODO(manu): We should have a more elaborate zod schema for @@ -118,8 +147,6 @@ export const spanComponentsV3Schema = z ]), ); -export type SpanComponentsV3Data = z.infer; - export class SpanComponentsV3 { constructor(public data: SpanComponentsV3Data) {} diff --git a/js/util/span_identifier_v4.ts b/js/util/span_identifier_v4.ts index e92e3eddf..771d72aec 100644 --- a/js/util/span_identifier_v4.ts +++ b/js/util/span_identifier_v4.ts @@ -5,6 +5,7 @@ import { SpanComponentsV3, SpanObjectTypeV3, spanObjectTypeV3EnumSchema, + type SpanComponentsV3Data, } from "./span_identifier_v3"; import { ParentExperimentIds, @@ -19,7 +20,7 @@ import { uint8ArrayToString, } from "./bytes"; import { z } from "zod/v3"; -import { InvokeFunctionType as InvokeFunctionRequest } from "./generated_types"; +import type { InvokeFunctionType as InvokeFunctionRequest } from "./generated_plain_types"; import { mergeDicts } from "./object_util"; const ENCODING_VERSION_NUMBER_V4 = 4; @@ -88,7 +89,9 @@ const FIELDS_ID_TO_NAME: Record = { [Fields.ROOT_SPAN_ID]: "root_span_id", }; -export const spanComponentsV4Schema = z +export type SpanComponentsV4Data = SpanComponentsV3Data; + +export const spanComponentsV4Schema: z.ZodType = z .object({ object_type: spanObjectTypeV3EnumSchema, propagated_event: z.record(z.unknown()).nullish(), @@ -122,8 +125,6 @@ export const spanComponentsV4Schema = z ]), ); -export type SpanComponentsV4Data = z.infer; - export class SpanComponentsV4 { constructor(public data: SpanComponentsV4Data) {} From 55dd6c95a73195b5c44b5cc8381b5c7ce9d8f1f5 Mon Sep 17 00:00:00 2001 From: Luca Forstner Date: Wed, 26 Aug 2026 20:09:43 +0200 Subject: [PATCH 6/8] ref!: Remove plugins as part of public API (#2386) ref [braintrustdata/braintrust-sdk-javascript#1841](https://github.com/braintrustdata/braintrust-sdk-javascript/issues/1841) Removes Plugins and unsubscribe functionality. Reasoning: Plugins are a bit of an unnecessary API to be exposed because they just attach stuff and un-attach stuff. The user can very easily just manage this by themselves. Also, unsubscribe is literally called from nowhere in the SDK. --- .changeset/remove-plugins-public-api.md | 5 + .../auto-instrumentations/configs/ai-sdk.ts | 4 +- .../configs/anthropic.ts | 2 +- .../configs/bedrock-runtime.ts | 2 +- .../configs/claude-agent-sdk.ts | 2 +- .../configs/cloudflare-agents.ts | 2 +- .../configs/cloudflare-ai-chat.ts | 2 +- .../configs/cloudflare-think.ts | 2 +- .../auto-instrumentations/configs/cohere.ts | 2 +- .../configs/cursor-sdk.ts | 2 +- .../auto-instrumentations/configs/genkit.ts | 2 +- .../configs/github-copilot.ts | 2 +- .../configs/google-adk.ts | 2 +- .../configs/google-genai.ts | 2 +- js/src/auto-instrumentations/configs/groq.ts | 2 +- .../configs/huggingface-transformers.ts | 2 +- .../configs/huggingface.ts | 2 +- .../configs/langchain.ts | 2 +- .../configs/langsmith.ts | 2 +- .../auto-instrumentations/configs/mistral.ts | 2 +- .../auto-instrumentations/configs/ollama.ts | 2 +- .../configs/openai-agents.ts | 2 +- .../configs/openai-codex.ts | 2 +- .../auto-instrumentations/configs/openai.ts | 2 +- .../configs/openrouter-agent.ts | 2 +- .../configs/openrouter.ts | 2 +- .../configs/pi-coding-agent.ts | 2 +- .../configs/strands-agent-sdk.ts | 2 +- .../auto-instrumentations/configs/voyageai.ts | 2 +- .../auto-instrumentations/loader/cjs-patch.ts | 2 +- .../loader/special-case-patches.ts | 4 +- js/src/exports.ts | 2 +- js/src/imports.test.ts | 6 +- js/src/instrumentation/README.md | 71 +- .../auto-instrumentation-suppression.ts | 8 +- .../instrumentation/braintrust-plugin.test.ts | 1791 ----------------- js/src/instrumentation/braintrust-plugin.ts | 402 ---- .../core/channel-tracing.test.ts | 225 +-- .../instrumentation/core/channel-tracing.ts | 45 +- js/src/instrumentation/core/channel.ts | 55 - js/src/instrumentation/core/index.ts | 26 +- js/src/instrumentation/core/plugin.ts | 505 ----- js/src/instrumentation/core/types.ts | 89 +- js/src/instrumentation/index.test.ts | 13 + js/src/instrumentation/index.ts | 34 +- .../instrumentation-consumers.test.ts | 215 ++ .../instrumentation-consumers.ts | 159 ++ .../plugins/cloudflare-agents-plugin.ts | 140 -- .../instrumentation/plugins/openai-plugin.ts | 613 ------ .../{plugins => providers}/ai-sdk-channels.ts | 0 .../ai-sdk-instrumentation.streaming.test.ts} | 229 +-- .../ai-sdk-instrumentation.test.ts} | 80 +- .../ai-sdk-instrumentation.ts} | 859 ++++---- .../ai-sdk-v7-telemetry.test.ts | 0 .../anthropic-channels.ts | 0 .../anthropic-instrumentation.test.ts} | 2 +- .../anthropic-instrumentation.ts} | 45 +- ...nthropic-sessions-instrumentation.test.ts} | 11 +- .../bedrock-runtime-channels.ts | 0 .../bedrock-runtime-common.ts | 0 .../bedrock-runtime-instrumentation.test.ts} | 6 +- .../bedrock-runtime-instrumentation.ts} | 29 +- .../claude-agent-sdk-channels.ts | 0 ...ude-agent-sdk-instrumentation-constants.ts | 0 ...ent-sdk-instrumentation.streaming.test.ts} | 2 +- .../claude-agent-sdk-instrumentation.test.ts} | 122 +- .../claude-agent-sdk-instrumentation.ts} | 25 +- .../claude-agent-sdk-local-tool-context.ts | 0 .../claude-agent-sdk-local-tool-spans.ts | 0 .../cloudflare-agents-channels.ts | 0 ...cloudflare-agents-instrumentation.test.ts} | 26 +- .../cloudflare-agents-instrumentation.ts | 129 ++ .../cloudflare-ai-chat-channels.ts | 0 .../cloudflare-ai-chat-consumer.test.ts} | 28 +- .../cloudflare-ai-chat-consumer.ts} | 40 +- .../cloudflare-ai-chat-instrumentation.ts | 0 .../cloudflare-think-channels.ts | 0 .../cloudflare-think-context.ts | 0 .../cloudflare-think-instrumentation.test.ts} | 2 +- .../cloudflare-think-instrumentation.ts} | 35 +- .../{plugins => providers}/cohere-channels.ts | 0 .../cohere-instrumentation.test.ts} | 2 +- .../cohere-instrumentation.ts} | 129 +- .../cursor-sdk-channels.ts | 0 .../cursor-sdk-instrumentation.test.ts} | 18 +- .../cursor-sdk-instrumentation.ts} | 32 +- .../eve-instrumentation.test.ts} | 12 +- .../eve-instrumentation.ts} | 0 .../flue-instrumentation.test.ts} | 2 +- .../flue-instrumentation.ts} | 0 .../{plugins => providers}/genkit-channels.ts | 0 .../genkit-instrumentation.test.ts} | 13 +- .../genkit-instrumentation.ts} | 143 +- .../github-copilot-channels.ts | 0 .../github-copilot-instrumentation.test.ts} | 40 +- .../github-copilot-instrumentation.ts} | 29 +- .../google-adk-channels.ts | 0 .../google-adk-instrumentation.test.ts} | 78 +- .../google-adk-instrumentation.ts} | 59 +- .../google-genai-channels.ts | 0 .../google-genai-instrumentation.test.ts} | 63 +- .../google-genai-instrumentation.ts} | 176 +- .../{plugins => providers}/groq-channels.ts | 0 .../groq-instrumentation.test.ts} | 2 +- .../groq-instrumentation.ts} | 92 +- .../harness-agent-context.test.ts | 0 .../huggingface-channels.ts | 0 .../huggingface-instrumentation.test.ts} | 2 +- .../huggingface-instrumentation.ts} | 102 +- .../huggingface-transformers-channels.ts | 0 ...face-transformers-instrumentation.test.ts} | 4 +- ...ggingface-transformers-instrumentation.ts} | 78 +- .../instrumentation-names.test.ts | 0 .../langchain-channels.ts | 0 .../langchain-instrumentation.test.ts} | 31 +- .../langchain-instrumentation.ts} | 25 +- .../langsmith-channels.ts | 0 .../langsmith-instrumentation.test.ts} | 41 +- .../langsmith-instrumentation.ts} | 36 +- .../mistral-channels.ts | 0 .../mistral-instrumentation.test.ts} | 2 +- .../mistral-instrumentation.ts} | 275 ++- .../{plugins => providers}/ollama-channels.ts | 0 .../ollama-instrumentation.test.ts} | 2 +- .../ollama-instrumentation.ts} | 72 +- .../openai-agents-channels.ts | 0 .../openai-agents-instrumentation.test.ts} | 13 +- .../openai-agents-instrumentation.ts} | 31 +- .../openai-agents-trace-processor.ts | 0 .../{plugins => providers}/openai-channels.ts | 0 .../openai-codex-channels.ts | 0 .../openai-codex-instrumentation.test.ts} | 8 +- .../openai-codex-instrumentation.ts} | 29 +- .../openai-instrumentation.test.ts} | 2 +- .../providers/openai-instrumentation.ts | 573 ++++++ .../openrouter-agent-channels.ts | 0 .../openrouter-agent-instrumentation.test.ts} | 6 +- .../openrouter-agent-instrumentation.ts} | 178 +- .../openrouter-channels.ts | 0 .../openrouter-instrumentation.test.ts} | 6 +- .../openrouter-instrumentation.ts} | 437 ++-- .../pi-coding-agent-channels.ts | 0 .../pi-coding-agent-instrumentation.test.ts} | 54 +- .../pi-coding-agent-instrumentation.ts} | 114 +- .../strands-agent-sdk-channels.ts | 0 ...strands-agent-sdk-instrumentation.test.ts} | 30 +- .../strands-agent-sdk-instrumentation.ts} | 37 +- .../voyageai-channels.ts | 0 .../voyageai-instrumentation.test.ts} | 4 +- .../voyageai-instrumentation.ts} | 77 +- js/src/instrumentation/registry.test.ts | 46 +- js/src/instrumentation/registry.ts | 46 +- js/src/wrappers/ai-sdk/ai-sdk.ts | 2 +- .../wrappers/ai-sdk/harness-agent-context.ts | 7 +- js/src/wrappers/ai-sdk/telemetry.ts | 2 +- js/src/wrappers/anthropic.ts | 2 +- js/src/wrappers/bedrock-runtime.ts | 4 +- .../claude-agent-sdk/claude-agent-sdk.ts | 6 +- js/src/wrappers/cloudflare-agent.ts | 2 +- js/src/wrappers/cloudflare-ai-chat.ts | 2 +- js/src/wrappers/cloudflare-think.ts | 2 +- js/src/wrappers/cohere.ts | 2 +- js/src/wrappers/cursor-sdk.ts | 2 +- js/src/wrappers/genkit.test.ts | 2 +- js/src/wrappers/genkit.ts | 2 +- js/src/wrappers/github-copilot.ts | 2 +- js/src/wrappers/google-adk.ts | 2 +- js/src/wrappers/google-genai.ts | 2 +- js/src/wrappers/groq.ts | 2 +- js/src/wrappers/huggingface-transformers.ts | 2 +- js/src/wrappers/huggingface.ts | 2 +- js/src/wrappers/langsmith.ts | 2 +- js/src/wrappers/mistral.ts | 2 +- js/src/wrappers/oai.ts | 2 +- js/src/wrappers/oai_responses.ts | 2 +- js/src/wrappers/ollama.test.ts | 2 +- js/src/wrappers/ollama.ts | 2 +- js/src/wrappers/openai-codex.ts | 2 +- js/src/wrappers/openai-promise-utils.ts | 2 +- js/src/wrappers/openrouter-agent.test.ts | 2 +- js/src/wrappers/openrouter-agent.ts | 2 +- js/src/wrappers/openrouter.ts | 2 +- js/src/wrappers/pi-coding-agent.ts | 2 +- js/src/wrappers/strands-agent-sdk.ts | 2 +- js/src/wrappers/voyageai.ts | 2 +- 185 files changed, 3059 insertions(+), 6407 deletions(-) create mode 100644 .changeset/remove-plugins-public-api.md delete mode 100644 js/src/instrumentation/braintrust-plugin.test.ts delete mode 100644 js/src/instrumentation/braintrust-plugin.ts delete mode 100644 js/src/instrumentation/core/channel.ts delete mode 100644 js/src/instrumentation/core/plugin.ts create mode 100644 js/src/instrumentation/index.test.ts create mode 100644 js/src/instrumentation/instrumentation-consumers.test.ts create mode 100644 js/src/instrumentation/instrumentation-consumers.ts delete mode 100644 js/src/instrumentation/plugins/cloudflare-agents-plugin.ts delete mode 100644 js/src/instrumentation/plugins/openai-plugin.ts rename js/src/instrumentation/{plugins => providers}/ai-sdk-channels.ts (100%) rename js/src/instrumentation/{plugins/ai-sdk-plugin.streaming.test.ts => providers/ai-sdk-instrumentation.streaming.test.ts} (92%) rename js/src/instrumentation/{plugins/ai-sdk-plugin.test.ts => providers/ai-sdk-instrumentation.test.ts} (96%) rename js/src/instrumentation/{plugins/ai-sdk-plugin.ts => providers/ai-sdk-instrumentation.ts} (85%) rename js/src/instrumentation/{plugins => providers}/ai-sdk-v7-telemetry.test.ts (100%) rename js/src/instrumentation/{plugins => providers}/anthropic-channels.ts (100%) rename js/src/instrumentation/{plugins/anthropic-plugin.test.ts => providers/anthropic-instrumentation.test.ts} (99%) rename js/src/instrumentation/{plugins/anthropic-plugin.ts => providers/anthropic-instrumentation.ts} (98%) rename js/src/instrumentation/{plugins/anthropic-sessions-plugin.test.ts => providers/anthropic-sessions-instrumentation.test.ts} (97%) rename js/src/instrumentation/{plugins => providers}/bedrock-runtime-channels.ts (100%) rename js/src/instrumentation/{plugins => providers}/bedrock-runtime-common.ts (100%) rename js/src/instrumentation/{plugins/bedrock-runtime-plugin.test.ts => providers/bedrock-runtime-instrumentation.test.ts} (99%) rename js/src/instrumentation/{plugins/bedrock-runtime-plugin.ts => providers/bedrock-runtime-instrumentation.ts} (97%) rename js/src/instrumentation/{plugins => providers}/claude-agent-sdk-channels.ts (100%) rename js/src/instrumentation/{plugins => providers}/claude-agent-sdk-instrumentation-constants.ts (100%) rename js/src/instrumentation/{plugins/claude-agent-sdk-plugin.streaming.test.ts => providers/claude-agent-sdk-instrumentation.streaming.test.ts} (99%) rename js/src/instrumentation/{plugins/claude-agent-sdk-plugin.test.ts => providers/claude-agent-sdk-instrumentation.test.ts} (86%) rename js/src/instrumentation/{plugins/claude-agent-sdk-plugin.ts => providers/claude-agent-sdk-instrumentation.ts} (99%) rename js/src/instrumentation/{plugins => providers}/claude-agent-sdk-local-tool-context.ts (100%) rename js/src/instrumentation/{plugins => providers}/claude-agent-sdk-local-tool-spans.ts (100%) rename js/src/instrumentation/{plugins => providers}/cloudflare-agents-channels.ts (100%) rename js/src/instrumentation/{plugins/cloudflare-agents-plugin.test.ts => providers/cloudflare-agents-instrumentation.test.ts} (88%) create mode 100644 js/src/instrumentation/providers/cloudflare-agents-instrumentation.ts rename js/src/instrumentation/{plugins => providers}/cloudflare-ai-chat-channels.ts (100%) rename js/src/instrumentation/{plugins/cloudflare-ai-chat-plugin.test.ts => providers/cloudflare-ai-chat-consumer.test.ts} (94%) rename js/src/instrumentation/{plugins/cloudflare-ai-chat-plugin.ts => providers/cloudflare-ai-chat-consumer.ts} (94%) rename js/src/instrumentation/{plugins => providers}/cloudflare-ai-chat-instrumentation.ts (100%) rename js/src/instrumentation/{plugins => providers}/cloudflare-think-channels.ts (100%) rename js/src/instrumentation/{plugins => providers}/cloudflare-think-context.ts (100%) rename js/src/instrumentation/{plugins/cloudflare-think-plugin.test.ts => providers/cloudflare-think-instrumentation.test.ts} (98%) rename js/src/instrumentation/{plugins/cloudflare-think-plugin.ts => providers/cloudflare-think-instrumentation.ts} (93%) rename js/src/instrumentation/{plugins => providers}/cohere-channels.ts (100%) rename js/src/instrumentation/{plugins/cohere-plugin.test.ts => providers/cohere-instrumentation.test.ts} (99%) rename js/src/instrumentation/{plugins/cohere-plugin.ts => providers/cohere-instrumentation.ts} (89%) rename js/src/instrumentation/{plugins => providers}/cursor-sdk-channels.ts (100%) rename js/src/instrumentation/{plugins/cursor-sdk-plugin.test.ts => providers/cursor-sdk-instrumentation.test.ts} (94%) rename js/src/instrumentation/{plugins/cursor-sdk-plugin.ts => providers/cursor-sdk-instrumentation.ts} (98%) rename js/src/instrumentation/{plugins/eve-plugin.test.ts => providers/eve-instrumentation.test.ts} (99%) rename js/src/instrumentation/{plugins/eve-plugin.ts => providers/eve-instrumentation.ts} (100%) rename js/src/instrumentation/{plugins/flue-plugin.test.ts => providers/flue-instrumentation.test.ts} (99%) rename js/src/instrumentation/{plugins/flue-plugin.ts => providers/flue-instrumentation.ts} (100%) rename js/src/instrumentation/{plugins => providers}/genkit-channels.ts (100%) rename js/src/instrumentation/{plugins/genkit-plugin.test.ts => providers/genkit-instrumentation.test.ts} (91%) rename js/src/instrumentation/{plugins/genkit-plugin.ts => providers/genkit-instrumentation.ts} (85%) rename js/src/instrumentation/{plugins => providers}/github-copilot-channels.ts (100%) rename js/src/instrumentation/{plugins/github-copilot-plugin.test.ts => providers/github-copilot-instrumentation.test.ts} (79%) rename js/src/instrumentation/{plugins/github-copilot-plugin.ts => providers/github-copilot-instrumentation.ts} (97%) rename js/src/instrumentation/{plugins => providers}/google-adk-channels.ts (100%) rename js/src/instrumentation/{plugins/google-adk-plugin.test.ts => providers/google-adk-instrumentation.test.ts} (91%) rename js/src/instrumentation/{plugins/google-adk-plugin.ts => providers/google-adk-instrumentation.ts} (95%) rename js/src/instrumentation/{plugins => providers}/google-genai-channels.ts (100%) rename js/src/instrumentation/{plugins/google-genai-plugin.test.ts => providers/google-genai-instrumentation.test.ts} (95%) rename js/src/instrumentation/{plugins/google-genai-plugin.ts => providers/google-genai-instrumentation.ts} (92%) rename js/src/instrumentation/{plugins => providers}/groq-channels.ts (100%) rename js/src/instrumentation/{plugins/groq-plugin.test.ts => providers/groq-instrumentation.test.ts} (98%) rename js/src/instrumentation/{plugins/groq-plugin.ts => providers/groq-instrumentation.ts} (58%) rename js/src/instrumentation/{plugins => providers}/harness-agent-context.test.ts (100%) rename js/src/instrumentation/{plugins => providers}/huggingface-channels.ts (100%) rename js/src/instrumentation/{plugins/huggingface-plugin.test.ts => providers/huggingface-instrumentation.test.ts} (98%) rename js/src/instrumentation/{plugins/huggingface-plugin.ts => providers/huggingface-instrumentation.ts} (87%) rename js/src/instrumentation/{plugins => providers}/huggingface-transformers-channels.ts (100%) rename js/src/instrumentation/{plugins/huggingface-transformers-plugin.test.ts => providers/huggingface-transformers-instrumentation.test.ts} (97%) rename js/src/instrumentation/{plugins/huggingface-transformers-plugin.ts => providers/huggingface-transformers-instrumentation.ts} (82%) rename js/src/instrumentation/{plugins => providers}/instrumentation-names.test.ts (100%) rename js/src/instrumentation/{plugins => providers}/langchain-channels.ts (100%) rename js/src/instrumentation/{plugins/langchain-plugin.test.ts => providers/langchain-instrumentation.test.ts} (79%) rename js/src/instrumentation/{plugins/langchain-plugin.ts => providers/langchain-instrumentation.ts} (89%) rename js/src/instrumentation/{plugins => providers}/langsmith-channels.ts (100%) rename js/src/instrumentation/{plugins/langsmith-plugin.test.ts => providers/langsmith-instrumentation.test.ts} (88%) rename js/src/instrumentation/{plugins/langsmith-plugin.ts => providers/langsmith-instrumentation.ts} (95%) rename js/src/instrumentation/{plugins => providers}/mistral-channels.ts (100%) rename js/src/instrumentation/{plugins/mistral-plugin.test.ts => providers/mistral-instrumentation.test.ts} (99%) rename js/src/instrumentation/{plugins/mistral-plugin.ts => providers/mistral-instrumentation.ts} (77%) rename js/src/instrumentation/{plugins => providers}/ollama-channels.ts (100%) rename js/src/instrumentation/{plugins/ollama-plugin.test.ts => providers/ollama-instrumentation.test.ts} (99%) rename js/src/instrumentation/{plugins/ollama-plugin.ts => providers/ollama-instrumentation.ts} (91%) rename js/src/instrumentation/{plugins => providers}/openai-agents-channels.ts (100%) rename js/src/instrumentation/{plugins/openai-agents-plugin.test.ts => providers/openai-agents-instrumentation.test.ts} (88%) rename js/src/instrumentation/{plugins/openai-agents-plugin.ts => providers/openai-agents-instrumentation.ts} (80%) rename js/src/instrumentation/{plugins => providers}/openai-agents-trace-processor.ts (100%) rename js/src/instrumentation/{plugins => providers}/openai-channels.ts (100%) rename js/src/instrumentation/{plugins => providers}/openai-codex-channels.ts (100%) rename js/src/instrumentation/{plugins/openai-codex-plugin.test.ts => providers/openai-codex-instrumentation.test.ts} (92%) rename js/src/instrumentation/{plugins/openai-codex-plugin.ts => providers/openai-codex-instrumentation.ts} (98%) rename js/src/instrumentation/{plugins/openai-plugin.test.ts => providers/openai-instrumentation.test.ts} (99%) create mode 100644 js/src/instrumentation/providers/openai-instrumentation.ts rename js/src/instrumentation/{plugins => providers}/openrouter-agent-channels.ts (100%) rename js/src/instrumentation/{plugins/openrouter-agent-plugin.test.ts => providers/openrouter-agent-instrumentation.test.ts} (99%) rename js/src/instrumentation/{plugins/openrouter-agent-plugin.ts => providers/openrouter-agent-instrumentation.ts} (91%) rename js/src/instrumentation/{plugins => providers}/openrouter-channels.ts (100%) rename js/src/instrumentation/{plugins/openrouter-plugin.test.ts => providers/openrouter-instrumentation.test.ts} (99%) rename js/src/instrumentation/{plugins/openrouter-plugin.ts => providers/openrouter-instrumentation.ts} (80%) rename js/src/instrumentation/{plugins => providers}/pi-coding-agent-channels.ts (100%) rename js/src/instrumentation/{plugins/pi-coding-agent-plugin.test.ts => providers/pi-coding-agent-instrumentation.test.ts} (91%) rename js/src/instrumentation/{plugins/pi-coding-agent-plugin.ts => providers/pi-coding-agent-instrumentation.ts} (93%) rename js/src/instrumentation/{plugins => providers}/strands-agent-sdk-channels.ts (100%) rename js/src/instrumentation/{plugins/strands-agent-sdk-plugin.test.ts => providers/strands-agent-sdk-instrumentation.test.ts} (95%) rename js/src/instrumentation/{plugins/strands-agent-sdk-plugin.ts => providers/strands-agent-sdk-instrumentation.ts} (97%) rename js/src/instrumentation/{plugins => providers}/voyageai-channels.ts (100%) rename js/src/instrumentation/{plugins/voyageai-plugin.test.ts => providers/voyageai-instrumentation.test.ts} (98%) rename js/src/instrumentation/{plugins/voyageai-plugin.ts => providers/voyageai-instrumentation.ts} (89%) diff --git a/.changeset/remove-plugins-public-api.md b/.changeset/remove-plugins-public-api.md new file mode 100644 index 000000000..21275cc2e --- /dev/null +++ b/.changeset/remove-plugins-public-api.md @@ -0,0 +1,5 @@ +--- +"braintrust": major +--- + +ref!: Remove plugins as part of public API diff --git a/js/src/auto-instrumentations/configs/ai-sdk.ts b/js/src/auto-instrumentations/configs/ai-sdk.ts index cdbb0c1f3..d38234e5e 100644 --- a/js/src/auto-instrumentations/configs/ai-sdk.ts +++ b/js/src/auto-instrumentations/configs/ai-sdk.ts @@ -2,7 +2,7 @@ import type { InstrumentationConfig } from "../orchestrion-js"; import { aiSDKChannels, harnessAgentChannels, -} from "../../instrumentation/plugins/ai-sdk-channels"; +} from "../../instrumentation/providers/ai-sdk-channels"; /** * Instrumentation configurations for the Vercel AI SDK. @@ -222,7 +222,7 @@ export const aiSDKConfigs: InstrumentationConfig[] = [ }, // AI SDK v7 exposes its telemetry lifecycle through a dispatcher created for - // each operation. We patch that dispatcher in the plugin instead of rewriting + // each operation. We patch that dispatcher in the consumer instead of rewriting // the module to call registerTelemetry(). { channelName: aiSDKChannels.v7CreateTelemetryDispatcher.channelName, diff --git a/js/src/auto-instrumentations/configs/anthropic.ts b/js/src/auto-instrumentations/configs/anthropic.ts index 3acaa4d75..862e3528b 100644 --- a/js/src/auto-instrumentations/configs/anthropic.ts +++ b/js/src/auto-instrumentations/configs/anthropic.ts @@ -1,5 +1,5 @@ import type { InstrumentationConfig } from "../orchestrion-js"; -import { anthropicChannels } from "../../instrumentation/plugins/anthropic-channels"; +import { anthropicChannels } from "../../instrumentation/providers/anthropic-channels"; /** * Instrumentation configurations for the Anthropic SDK. diff --git a/js/src/auto-instrumentations/configs/bedrock-runtime.ts b/js/src/auto-instrumentations/configs/bedrock-runtime.ts index 29b4d0f0d..91dddcfae 100644 --- a/js/src/auto-instrumentations/configs/bedrock-runtime.ts +++ b/js/src/auto-instrumentations/configs/bedrock-runtime.ts @@ -2,7 +2,7 @@ import type { InstrumentationConfig } from "@apm-js-collab/code-transformer"; import { smithyClientChannels, smithyCoreChannels, -} from "../../instrumentation/plugins/bedrock-runtime-channels"; +} from "../../instrumentation/providers/bedrock-runtime-channels"; export const bedrockRuntimeConfigs: InstrumentationConfig[] = [ { diff --git a/js/src/auto-instrumentations/configs/claude-agent-sdk.ts b/js/src/auto-instrumentations/configs/claude-agent-sdk.ts index 1cf2c6537..6b663df61 100644 --- a/js/src/auto-instrumentations/configs/claude-agent-sdk.ts +++ b/js/src/auto-instrumentations/configs/claude-agent-sdk.ts @@ -1,5 +1,5 @@ import type { InstrumentationConfig } from "../orchestrion-js"; -import { claudeAgentSDKChannels } from "../../instrumentation/plugins/claude-agent-sdk-channels"; +import { claudeAgentSDKChannels } from "../../instrumentation/providers/claude-agent-sdk-channels"; /** * Instrumentation configuration for the Claude Agent SDK. diff --git a/js/src/auto-instrumentations/configs/cloudflare-agents.ts b/js/src/auto-instrumentations/configs/cloudflare-agents.ts index 0b655bb78..b18372cb1 100644 --- a/js/src/auto-instrumentations/configs/cloudflare-agents.ts +++ b/js/src/auto-instrumentations/configs/cloudflare-agents.ts @@ -1,5 +1,5 @@ import type { InstrumentationConfig } from "../orchestrion-js"; -import { cloudflareAgentsChannels } from "../../instrumentation/plugins/cloudflare-agents-channels"; +import { cloudflareAgentsChannels } from "../../instrumentation/providers/cloudflare-agents-channels"; const cloudflareAgentsVersionRange = ">=0.17.0 <0.18.0"; diff --git a/js/src/auto-instrumentations/configs/cloudflare-ai-chat.ts b/js/src/auto-instrumentations/configs/cloudflare-ai-chat.ts index 02f081612..1a603bffc 100644 --- a/js/src/auto-instrumentations/configs/cloudflare-ai-chat.ts +++ b/js/src/auto-instrumentations/configs/cloudflare-ai-chat.ts @@ -1,5 +1,5 @@ import type { InstrumentationConfig } from "../orchestrion-js"; -import { cloudflareAIChatChannels } from "../../instrumentation/plugins/cloudflare-ai-chat-channels"; +import { cloudflareAIChatChannels } from "../../instrumentation/providers/cloudflare-ai-chat-channels"; const cloudflareAIChatVersionRange = ">=0.9.0 <0.10.0"; diff --git a/js/src/auto-instrumentations/configs/cloudflare-think.ts b/js/src/auto-instrumentations/configs/cloudflare-think.ts index 61fb4e8ae..d0e270b0f 100644 --- a/js/src/auto-instrumentations/configs/cloudflare-think.ts +++ b/js/src/auto-instrumentations/configs/cloudflare-think.ts @@ -1,5 +1,5 @@ import type { InstrumentationConfig } from "../orchestrion-js"; -import { cloudflareThinkChannels } from "../../instrumentation/plugins/cloudflare-think-channels"; +import { cloudflareThinkChannels } from "../../instrumentation/providers/cloudflare-think-channels"; const cloudflareThinkVersionRange = ">=0.13.0 <0.14.0"; diff --git a/js/src/auto-instrumentations/configs/cohere.ts b/js/src/auto-instrumentations/configs/cohere.ts index 106980884..1ad936ac7 100644 --- a/js/src/auto-instrumentations/configs/cohere.ts +++ b/js/src/auto-instrumentations/configs/cohere.ts @@ -1,5 +1,5 @@ import type { InstrumentationConfig } from "../orchestrion-js"; -import { cohereChannels } from "../../instrumentation/plugins/cohere-channels"; +import { cohereChannels } from "../../instrumentation/providers/cohere-channels"; export const cohereConfigs: InstrumentationConfig[] = [ { diff --git a/js/src/auto-instrumentations/configs/cursor-sdk.ts b/js/src/auto-instrumentations/configs/cursor-sdk.ts index ecb845b4b..0b0eddb32 100644 --- a/js/src/auto-instrumentations/configs/cursor-sdk.ts +++ b/js/src/auto-instrumentations/configs/cursor-sdk.ts @@ -1,5 +1,5 @@ import type { InstrumentationConfig } from "../orchestrion-js"; -import { cursorSDKChannels } from "../../instrumentation/plugins/cursor-sdk-channels"; +import { cursorSDKChannels } from "../../instrumentation/providers/cursor-sdk-channels"; const cursorSDKVersionRange = ">=1.0.7 <2.0.0"; diff --git a/js/src/auto-instrumentations/configs/genkit.ts b/js/src/auto-instrumentations/configs/genkit.ts index 349635b01..5422a08fe 100644 --- a/js/src/auto-instrumentations/configs/genkit.ts +++ b/js/src/auto-instrumentations/configs/genkit.ts @@ -2,7 +2,7 @@ import type { InstrumentationConfig } from "../orchestrion-js"; import { genkitChannels, genkitCoreChannels, -} from "../../instrumentation/plugins/genkit-channels"; +} from "../../instrumentation/providers/genkit-channels"; const genkitVersionRange = ">=1.0.0 <2.0.0"; diff --git a/js/src/auto-instrumentations/configs/github-copilot.ts b/js/src/auto-instrumentations/configs/github-copilot.ts index 901f9e682..aed9eb7ed 100644 --- a/js/src/auto-instrumentations/configs/github-copilot.ts +++ b/js/src/auto-instrumentations/configs/github-copilot.ts @@ -1,5 +1,5 @@ import type { InstrumentationConfig } from "../orchestrion-js"; -import { gitHubCopilotChannels } from "../../instrumentation/plugins/github-copilot-channels"; +import { gitHubCopilotChannels } from "../../instrumentation/providers/github-copilot-channels"; export const gitHubCopilotConfigs: InstrumentationConfig[] = [ // ESM: CopilotClient.createSession diff --git a/js/src/auto-instrumentations/configs/google-adk.ts b/js/src/auto-instrumentations/configs/google-adk.ts index aeb72c44b..8c07fdbd7 100644 --- a/js/src/auto-instrumentations/configs/google-adk.ts +++ b/js/src/auto-instrumentations/configs/google-adk.ts @@ -1,5 +1,5 @@ import type { InstrumentationConfig } from "../orchestrion-js"; -import { googleADKChannels } from "../../instrumentation/plugins/google-adk-channels"; +import { googleADKChannels } from "../../instrumentation/providers/google-adk-channels"; const googleADKVersionRange = ">=0.1.0"; const googleADKBundledIndexV06VersionRange = ">=0.6.1 <0.7.0"; diff --git a/js/src/auto-instrumentations/configs/google-genai.ts b/js/src/auto-instrumentations/configs/google-genai.ts index 16edd1d5c..53fb24d43 100644 --- a/js/src/auto-instrumentations/configs/google-genai.ts +++ b/js/src/auto-instrumentations/configs/google-genai.ts @@ -1,5 +1,5 @@ import type { InstrumentationConfig } from "../orchestrion-js"; -import { googleGenAIChannels } from "../../instrumentation/plugins/google-genai-channels"; +import { googleGenAIChannels } from "../../instrumentation/providers/google-genai-channels"; /** * Instrumentation configurations for the Google GenAI SDK. diff --git a/js/src/auto-instrumentations/configs/groq.ts b/js/src/auto-instrumentations/configs/groq.ts index 864b2a166..5db99b980 100644 --- a/js/src/auto-instrumentations/configs/groq.ts +++ b/js/src/auto-instrumentations/configs/groq.ts @@ -1,5 +1,5 @@ import type { InstrumentationConfig } from "../orchestrion-js"; -import { groqChannels } from "../../instrumentation/plugins/groq-channels"; +import { groqChannels } from "../../instrumentation/providers/groq-channels"; export const groqConfigs: InstrumentationConfig[] = [ { diff --git a/js/src/auto-instrumentations/configs/huggingface-transformers.ts b/js/src/auto-instrumentations/configs/huggingface-transformers.ts index dd951ac25..781aab990 100644 --- a/js/src/auto-instrumentations/configs/huggingface-transformers.ts +++ b/js/src/auto-instrumentations/configs/huggingface-transformers.ts @@ -1,5 +1,5 @@ import type { InstrumentationConfig } from "../orchestrion-js"; -import { huggingFaceTransformersChannels } from "../../instrumentation/plugins/huggingface-transformers-channels"; +import { huggingFaceTransformersChannels } from "../../instrumentation/providers/huggingface-transformers-channels"; const moduleFiles = [ { diff --git a/js/src/auto-instrumentations/configs/huggingface.ts b/js/src/auto-instrumentations/configs/huggingface.ts index f4c9bec06..53b4d011f 100644 --- a/js/src/auto-instrumentations/configs/huggingface.ts +++ b/js/src/auto-instrumentations/configs/huggingface.ts @@ -1,5 +1,5 @@ import type { InstrumentationConfig } from "../orchestrion-js"; -import { huggingFaceChannels } from "../../instrumentation/plugins/huggingface-channels"; +import { huggingFaceChannels } from "../../instrumentation/providers/huggingface-channels"; export const huggingFaceConfigs: InstrumentationConfig[] = [ { diff --git a/js/src/auto-instrumentations/configs/langchain.ts b/js/src/auto-instrumentations/configs/langchain.ts index e8108ed02..c1263a285 100644 --- a/js/src/auto-instrumentations/configs/langchain.ts +++ b/js/src/auto-instrumentations/configs/langchain.ts @@ -1,5 +1,5 @@ import type { InstrumentationConfig } from "../orchestrion-js"; -import { langChainChannels } from "../../instrumentation/plugins/langchain-channels"; +import { langChainChannels } from "../../instrumentation/providers/langchain-channels"; const langChainCoreVersionRange = ">=0.3.42"; const langChainCallbackManagerFilePath = "dist/callbacks/manager.js"; diff --git a/js/src/auto-instrumentations/configs/langsmith.ts b/js/src/auto-instrumentations/configs/langsmith.ts index 5ecc54f87..0904bc64c 100644 --- a/js/src/auto-instrumentations/configs/langsmith.ts +++ b/js/src/auto-instrumentations/configs/langsmith.ts @@ -1,5 +1,5 @@ import type { InstrumentationConfig } from "../orchestrion-js"; -import { langSmithChannels } from "../../instrumentation/plugins/langsmith-channels"; +import { langSmithChannels } from "../../instrumentation/providers/langsmith-channels"; const versionRange = ">=0.3.30 <1.0.0"; diff --git a/js/src/auto-instrumentations/configs/mistral.ts b/js/src/auto-instrumentations/configs/mistral.ts index e61fc24be..dab2493c3 100644 --- a/js/src/auto-instrumentations/configs/mistral.ts +++ b/js/src/auto-instrumentations/configs/mistral.ts @@ -1,5 +1,5 @@ import type { InstrumentationConfig } from "../orchestrion-js"; -import { mistralChannels } from "../../instrumentation/plugins/mistral-channels"; +import { mistralChannels } from "../../instrumentation/providers/mistral-channels"; export const mistralConfigs: InstrumentationConfig[] = [ { diff --git a/js/src/auto-instrumentations/configs/ollama.ts b/js/src/auto-instrumentations/configs/ollama.ts index 5ef28eb64..8403bb652 100644 --- a/js/src/auto-instrumentations/configs/ollama.ts +++ b/js/src/auto-instrumentations/configs/ollama.ts @@ -1,5 +1,5 @@ import type { InstrumentationConfig } from "../orchestrion-js"; -import { ollamaChannels } from "../../instrumentation/plugins/ollama-channels"; +import { ollamaChannels } from "../../instrumentation/providers/ollama-channels"; const methods = [ ["chat", ollamaChannels.chat.channelName], diff --git a/js/src/auto-instrumentations/configs/openai-agents.ts b/js/src/auto-instrumentations/configs/openai-agents.ts index 3647edbc5..8bf95e99f 100644 --- a/js/src/auto-instrumentations/configs/openai-agents.ts +++ b/js/src/auto-instrumentations/configs/openai-agents.ts @@ -1,5 +1,5 @@ import type { InstrumentationConfig } from "../orchestrion-js"; -import { openAIAgentsCoreChannels } from "../../instrumentation/plugins/openai-agents-channels"; +import { openAIAgentsCoreChannels } from "../../instrumentation/providers/openai-agents-channels"; const lifecycleMethods = [ ["onTraceStart", openAIAgentsCoreChannels.onTraceStart.channelName], diff --git a/js/src/auto-instrumentations/configs/openai-codex.ts b/js/src/auto-instrumentations/configs/openai-codex.ts index 40ee60784..a725caeb9 100644 --- a/js/src/auto-instrumentations/configs/openai-codex.ts +++ b/js/src/auto-instrumentations/configs/openai-codex.ts @@ -1,5 +1,5 @@ import type { InstrumentationConfig } from "../orchestrion-js"; -import { openAICodexChannels } from "../../instrumentation/plugins/openai-codex-channels"; +import { openAICodexChannels } from "../../instrumentation/providers/openai-codex-channels"; const openAICodexVersionRange = ">=0.128.0 <1.0.0"; diff --git a/js/src/auto-instrumentations/configs/openai.ts b/js/src/auto-instrumentations/configs/openai.ts index 28d85b7f6..411bd2654 100644 --- a/js/src/auto-instrumentations/configs/openai.ts +++ b/js/src/auto-instrumentations/configs/openai.ts @@ -1,5 +1,5 @@ import type { InstrumentationConfig } from "../orchestrion-js"; -import { openAIChannels } from "../../instrumentation/plugins/openai-channels"; +import { openAIChannels } from "../../instrumentation/providers/openai-channels"; /** * Instrumentation configurations for the OpenAI SDK. diff --git a/js/src/auto-instrumentations/configs/openrouter-agent.ts b/js/src/auto-instrumentations/configs/openrouter-agent.ts index f9f6191e1..d228c1ed8 100644 --- a/js/src/auto-instrumentations/configs/openrouter-agent.ts +++ b/js/src/auto-instrumentations/configs/openrouter-agent.ts @@ -1,5 +1,5 @@ import type { InstrumentationConfig } from "../orchestrion-js"; -import { openRouterAgentChannels } from "../../instrumentation/plugins/openrouter-agent-channels"; +import { openRouterAgentChannels } from "../../instrumentation/providers/openrouter-agent-channels"; export const openRouterAgentConfigs: InstrumentationConfig[] = [ { diff --git a/js/src/auto-instrumentations/configs/openrouter.ts b/js/src/auto-instrumentations/configs/openrouter.ts index c7b2ce7e4..2607b3da4 100644 --- a/js/src/auto-instrumentations/configs/openrouter.ts +++ b/js/src/auto-instrumentations/configs/openrouter.ts @@ -1,5 +1,5 @@ import type { InstrumentationConfig } from "../orchestrion-js"; -import { openRouterChannels } from "../../instrumentation/plugins/openrouter-channels"; +import { openRouterChannels } from "../../instrumentation/providers/openrouter-channels"; export const openRouterConfigs: InstrumentationConfig[] = [ { diff --git a/js/src/auto-instrumentations/configs/pi-coding-agent.ts b/js/src/auto-instrumentations/configs/pi-coding-agent.ts index 0bf6e2a56..3401aec0d 100644 --- a/js/src/auto-instrumentations/configs/pi-coding-agent.ts +++ b/js/src/auto-instrumentations/configs/pi-coding-agent.ts @@ -1,5 +1,5 @@ import type { InstrumentationConfig } from "../orchestrion-js"; -import { piCodingAgentChannels } from "../../instrumentation/plugins/pi-coding-agent-channels"; +import { piCodingAgentChannels } from "../../instrumentation/providers/pi-coding-agent-channels"; const piCodingAgentVersionRange = ">=0.79.0 <0.82.0"; diff --git a/js/src/auto-instrumentations/configs/strands-agent-sdk.ts b/js/src/auto-instrumentations/configs/strands-agent-sdk.ts index 7a7f7a5b9..f02439a19 100644 --- a/js/src/auto-instrumentations/configs/strands-agent-sdk.ts +++ b/js/src/auto-instrumentations/configs/strands-agent-sdk.ts @@ -1,5 +1,5 @@ import type { InstrumentationConfig } from "@apm-js-collab/code-transformer"; -import { strandsAgentSDKChannels } from "../../instrumentation/plugins/strands-agent-sdk-channels"; +import { strandsAgentSDKChannels } from "../../instrumentation/providers/strands-agent-sdk-channels"; const strandsAgentSDKVersionRange = ">=1.0.0 <2.0.0"; diff --git a/js/src/auto-instrumentations/configs/voyageai.ts b/js/src/auto-instrumentations/configs/voyageai.ts index f54edf84e..9fbe886be 100644 --- a/js/src/auto-instrumentations/configs/voyageai.ts +++ b/js/src/auto-instrumentations/configs/voyageai.ts @@ -1,5 +1,5 @@ import type { InstrumentationConfig } from "../orchestrion-js"; -import { voyageAIChannels } from "../../instrumentation/plugins/voyageai-channels"; +import { voyageAIChannels } from "../../instrumentation/providers/voyageai-channels"; const GENERATED_CLIENT_MODULES = [ "dist/cjs/Client.js", diff --git a/js/src/auto-instrumentations/loader/cjs-patch.ts b/js/src/auto-instrumentations/loader/cjs-patch.ts index df5c448ae..d94511563 100644 --- a/js/src/auto-instrumentations/loader/cjs-patch.ts +++ b/js/src/auto-instrumentations/loader/cjs-patch.ts @@ -50,7 +50,7 @@ export class ModulePatch { // Per-package source patches (see loader/special-case-patches.ts). // Anti-pattern intentionally isolated in its own module — do not - // expand inline here; new integrations belong in the standard plugin + // expand inline here; new integrations belong in the standard consumer // pipeline. const patched = applySpecialCasePatch({ packageName, diff --git a/js/src/auto-instrumentations/loader/special-case-patches.ts b/js/src/auto-instrumentations/loader/special-case-patches.ts index f42215592..bf94b5926 100644 --- a/js/src/auto-instrumentations/loader/special-case-patches.ts +++ b/js/src/auto-instrumentations/loader/special-case-patches.ts @@ -6,8 +6,8 @@ * Every entry in this file represents a target SDK that doesn't expose a * stable extension point we can hook through global instrumentation hooks + the * internal Orchestrion matcher. New integrations should - * **prefer the standard channel-handler / `BasePlugin` pattern** used by - * every other integration in `js/src/instrumentation/plugins/*-plugin.ts`. + * **prefer the standard channel-consumer pattern** used by the integrations in + * `js/src/instrumentation/providers/*-plugin.ts`. * Only add an entry here when the target SDK gives us no other option (e.g. * the SDK relies on side-effectful module initialization, exposes its API * exclusively as re-exports from content-hashed chunks, or otherwise can't diff --git a/js/src/exports.ts b/js/src/exports.ts index e8286a959..487ec7430 100644 --- a/js/src/exports.ts +++ b/js/src/exports.ts @@ -200,7 +200,7 @@ export { BraintrustMiddleware, wrapAISDKModel, } from "./wrappers/ai-sdk"; -export { braintrustEveInstrumentation } from "./instrumentation/plugins/eve-plugin"; +export { braintrustEveInstrumentation } from "./instrumentation"; export { collectAnthropicSession } from "./wrappers/anthropic-session-collector"; export { wrapAnthropic } from "./wrappers/anthropic"; export { diff --git a/js/src/imports.test.ts b/js/src/imports.test.ts index c2d53c767..e00c08223 100644 --- a/js/src/imports.test.ts +++ b/js/src/imports.test.ts @@ -133,11 +133,13 @@ describe("Import restrictions", () => { // Check for dynamic import() statements // Match import(...) but not static import statements - // Exception: allow dynamic import in anthropic-plugin for APIPromise patching + // Exception: allow dynamic import in anthropic-instrumentation for APIPromise patching if ( /\bimport\s*\(/.test(line) && !/^import\s+/.test(line.trim()) && - !relativePath.includes("instrumentation/plugins/anthropic-plugin.ts") + !relativePath.includes( + "instrumentation/providers/anthropic-instrumentation.ts", + ) ) { violations.push( `${relativePath}:${index + 1} - Found dynamic import() statement: "${line.trim()}"`, diff --git a/js/src/instrumentation/README.md b/js/src/instrumentation/README.md index 26487ec26..1fc368594 100644 --- a/js/src/instrumentation/README.md +++ b/js/src/instrumentation/README.md @@ -1,7 +1,8 @@ -# Writing Braintrust Instrumentation Plugins +# Braintrust Instrumentation Consumers -Braintrust instrumentation plugins wrap provider calls through typed invocation -hooks or consume tracing-compatible events from the internal global registry. +Braintrust instrumentation consumers wrap provider calls through typed +invocation hooks or consume tracing-compatible events from the internal global +registry. Auto-instrumented provider code and manual wrappers use the same typed channels, so extraction, stream handling, and span behavior stay aligned. @@ -13,15 +14,15 @@ An instrumentation has four parts: transformation. 2. A typed channel defines its arguments, result, extra event fields, and stable `orchestrion::` identifier. -3. A plugin intercepts that channel, or subscribes to its legacy tracing - lifecycle, and maps the call into Braintrust spans. +3. An internal consumer intercepts that channel, or subscribes to its legacy + tracing lifecycle, and maps the call into Braintrust spans. 4. A manual wrapper invokes the same typed channel when transformation is not available. -The global hook transport is internal. New and migrated plugins should prefer -the typed channel's `intercept` API. Existing plugins can continue using -`traceAsyncChannel`, `traceStreamingChannel`, `traceSyncStreamChannel`, or -`BasePlugin` helpers during the gradual migration. +The global hook transport and its consumers are internal. New and migrated +consumers should prefer the typed channel's `intercept` API. Existing consumers +can continue using `traceAsyncChannel`, `traceStreamingChannel`, or +`traceSyncStreamChannel` during the gradual migration. ## Invocation Hooks @@ -100,27 +101,25 @@ construct it from the package and operation. Prefer the shared tracing helpers: ```ts -this.register( - traceAsyncChannel(providerChannels.create, { - name: "provider.messages.create", - type: "llm", - extractInput(args) { - return { - input: args[0].messages, - metadata: { model: args[0].model }, - }; - }, - extractOutput(result) { - return result.content; - }, - extractMetrics(result) { - return { - prompt_tokens: result.usage.input_tokens, - completion_tokens: result.usage.output_tokens, - }; - }, - }), -); +traceAsyncChannel(providerChannels.create, { + name: "provider.messages.create", + type: "llm", + extractInput(args) { + return { + input: args[0].messages, + metadata: { model: args[0].model }, + }; + }, + extractOutput(result) { + return result.content; + }, + extractMetrics(result) { + return { + prompt_tokens: result.usage.input_tokens, + completion_tokens: result.usage.output_tokens, + }; + }, +}); ``` The helpers: @@ -129,7 +128,7 @@ The helpers: - bind the current span store to `start` for async-context propagation - contain extraction failures and log them through `debugLogger` - patch streams without replacing their public semantics -- unsubscribe and unbind stores when a plugin is disabled +- install process-lifetime subscriptions and span-store bindings Use raw `IsoChannelHandlers` only when a provider requires lifecycle behavior that the shared helpers cannot express. @@ -146,7 +145,7 @@ return providerChannels.create.invoke(originalCreate, this, [params], { ``` Legacy wrappers can continue calling the tracing-compatible operators until -their plugin is migrated: +their consumer is migrated: ```ts return providerChannels.create.tracePromise(() => originalCreate(params), { @@ -155,7 +154,7 @@ return providerChannels.create.tracePromise(() => originalCreate(params), { ``` Do not create spans directly inside wrappers. Keeping span creation in the -plugin prevents auto and manual instrumentation from drifting. +internal consumer prevents auto and manual instrumentation from drifting. ## Promise and Stream Requirements @@ -177,14 +176,14 @@ termination, and async context. - Avoid prototype-sensitive merges and unnecessary mutation of provider data. - Capture only fields permitted by the instrumentation specification. - Pass `Error` objects directly to `span.log({ error })`. -- Use narrow vendored provider interfaces shared by wrappers and plugins. -- Keep enable, disable, subscription, and patching behavior idempotent. +- Use narrow vendored provider interfaces shared by wrappers and consumers. +- Keep enable, subscription, and patching behavior idempotent. ## Testing Test at the narrowest useful layers: -1. Plugin unit tests for extraction and span handling. +1. Consumer unit tests for extraction and span handling. 2. Global hook/runtime tests for lifecycle and context behavior. 3. Orchestrion transformation tests for generated wrappers. 4. Bundler and loader tests for real transformed execution. diff --git a/js/src/instrumentation/auto-instrumentation-suppression.ts b/js/src/instrumentation/auto-instrumentation-suppression.ts index 2b31044ef..6002e0d1f 100644 --- a/js/src/instrumentation/auto-instrumentation-suppression.ts +++ b/js/src/instrumentation/auto-instrumentation-suppression.ts @@ -45,10 +45,10 @@ export function runWithAutoInstrumentationSuppressed(callback: () => R): R { export function bindAutoInstrumentationSuppressionToStart( tracingChannel: Pick, "start">, -): (() => void) | undefined { +): void { const startChannel = tracingChannel.start; if (!startChannel) { - return undefined; + return; } const store = suppressionStore(); @@ -61,10 +61,6 @@ export function bindAutoInstrumentationSuppressionToStart( }, ], })); - - return () => { - startChannel.unbindStore(store); - }; } export function enterAutoInstrumentationAllowed(): () => void { diff --git a/js/src/instrumentation/braintrust-plugin.test.ts b/js/src/instrumentation/braintrust-plugin.test.ts deleted file mode 100644 index 3365ec279..000000000 --- a/js/src/instrumentation/braintrust-plugin.test.ts +++ /dev/null @@ -1,1791 +0,0 @@ -import { describe, it, expect, vi, beforeEach } from "vitest"; -import { BraintrustPlugin } from "./braintrust-plugin"; -import { OpenAIPlugin } from "./plugins/openai-plugin"; -import { OpenAICodexPlugin } from "./plugins/openai-codex-plugin"; -import { AnthropicPlugin } from "./plugins/anthropic-plugin"; -import { AISDKPlugin } from "./plugins/ai-sdk-plugin"; -import { ClaudeAgentSDKPlugin } from "./plugins/claude-agent-sdk-plugin"; -import { CloudflareThinkPlugin } from "./plugins/cloudflare-think-plugin"; -import { OpenAIAgentsPlugin } from "./plugins/openai-agents-plugin"; -import { GoogleGenAIPlugin } from "./plugins/google-genai-plugin"; -import { HuggingFacePlugin } from "./plugins/huggingface-plugin"; -import { HuggingFaceTransformersPlugin } from "./plugins/huggingface-transformers-plugin"; -import { OpenRouterAgentPlugin } from "./plugins/openrouter-agent-plugin"; -import { OpenRouterPlugin } from "./plugins/openrouter-plugin"; -import { MistralPlugin } from "./plugins/mistral-plugin"; -import { OllamaPlugin } from "./plugins/ollama-plugin"; -import { CoherePlugin } from "./plugins/cohere-plugin"; -import { GroqPlugin } from "./plugins/groq-plugin"; -import { GitHubCopilotPlugin } from "./plugins/github-copilot-plugin"; -import { LangChainPlugin } from "./plugins/langchain-plugin"; -import { LangSmithPlugin } from "./plugins/langsmith-plugin"; -import { PiCodingAgentPlugin } from "./plugins/pi-coding-agent-plugin"; -import { StrandsAgentSDKPlugin } from "./plugins/strands-agent-sdk-plugin"; -import { VoyageAIPlugin } from "./plugins/voyageai-plugin"; -import { CloudflareAIChatPlugin } from "./plugins/cloudflare-ai-chat-plugin"; -import { CloudflareAgentsPlugin } from "./plugins/cloudflare-agents-plugin"; - -function createPluginClassMock() { - return vi.fn(function MockPlugin(this: { - enable: ReturnType; - disable: ReturnType; - }) { - this.enable = vi.fn(); - this.disable = vi.fn(); - }); -} - -// Preserve the re-exported utility functions (parseMetricsFromUsage, etc.) -// while mocking out the OpenAIPlugin class — those utilities are also -// imported and tested at the bottom of this file via braintrust-plugin's -// re-exports. -vi.mock("./plugins/openai-plugin", async () => { - const actual = await vi.importActual< - typeof import("./plugins/openai-plugin") - >("./plugins/openai-plugin"); - return { - ...actual, - OpenAIPlugin: createPluginClassMock(), - }; -}); - -vi.mock("./plugins/anthropic-plugin", () => ({ - AnthropicPlugin: createPluginClassMock(), -})); - -vi.mock("./plugins/openai-codex-plugin", () => ({ - OpenAICodexPlugin: createPluginClassMock(), -})); - -vi.mock("./plugins/ai-sdk-plugin", () => ({ - AISDKPlugin: createPluginClassMock(), -})); - -vi.mock("./plugins/claude-agent-sdk-plugin", () => ({ - ClaudeAgentSDKPlugin: createPluginClassMock(), -})); - -vi.mock("./plugins/cloudflare-think-plugin", () => ({ - CloudflareThinkPlugin: createPluginClassMock(), -})); - -vi.mock("./plugins/openai-agents-plugin", () => ({ - OpenAIAgentsPlugin: createPluginClassMock(), -})); - -vi.mock("./plugins/google-genai-plugin", () => ({ - GoogleGenAIPlugin: createPluginClassMock(), -})); - -vi.mock("./plugins/huggingface-plugin", () => ({ - HuggingFacePlugin: createPluginClassMock(), -})); - -vi.mock("./plugins/huggingface-transformers-plugin", () => ({ - HuggingFaceTransformersPlugin: createPluginClassMock(), -})); - -vi.mock("./plugins/openrouter-plugin", () => ({ - OpenRouterPlugin: createPluginClassMock(), -})); - -vi.mock("./plugins/openrouter-agent-plugin", () => ({ - OpenRouterAgentPlugin: createPluginClassMock(), -})); - -vi.mock("./plugins/mistral-plugin", () => ({ - MistralPlugin: createPluginClassMock(), -})); - -vi.mock("./plugins/ollama-plugin", () => ({ - OllamaPlugin: createPluginClassMock(), -})); - -vi.mock("./plugins/cohere-plugin", () => ({ - CoherePlugin: createPluginClassMock(), -})); - -vi.mock("./plugins/groq-plugin", () => ({ - GroqPlugin: createPluginClassMock(), -})); - -vi.mock("./plugins/github-copilot-plugin", () => ({ - GitHubCopilotPlugin: createPluginClassMock(), -})); - -vi.mock("./plugins/langchain-plugin", () => ({ - LangChainPlugin: createPluginClassMock(), -})); - -vi.mock("./plugins/langsmith-plugin", () => ({ - LangSmithPlugin: createPluginClassMock(), -})); - -vi.mock("./plugins/pi-coding-agent-plugin", () => ({ - PiCodingAgentPlugin: createPluginClassMock(), -})); - -vi.mock("./plugins/strands-agent-sdk-plugin", () => ({ - StrandsAgentSDKPlugin: createPluginClassMock(), -})); - -vi.mock("./plugins/voyageai-plugin", () => ({ - VoyageAIPlugin: createPluginClassMock(), -})); - -vi.mock("./plugins/cloudflare-ai-chat-plugin", () => ({ - CloudflareAIChatPlugin: createPluginClassMock(), -})); - -vi.mock("./plugins/cloudflare-agents-plugin", () => ({ - CloudflareAgentsPlugin: createPluginClassMock(), -})); - -describe("BraintrustPlugin", () => { - beforeEach(() => { - vi.clearAllMocks(); - }); - - describe("sub-plugin creation (default behavior)", () => { - it("should create and enable OpenAI plugin by default", () => { - const plugin = new BraintrustPlugin(); - plugin.enable(); - - expect(OpenAIPlugin).toHaveBeenCalledTimes(1); - const mockInstance = vi.mocked(OpenAIPlugin).mock.results[0].value; - expect(mockInstance.enable).toHaveBeenCalledTimes(1); - }); - - it("should create and enable Anthropic plugin by default", () => { - const plugin = new BraintrustPlugin(); - plugin.enable(); - - expect(AnthropicPlugin).toHaveBeenCalledTimes(1); - const mockInstance = vi.mocked(AnthropicPlugin).mock.results[0].value; - expect(mockInstance.enable).toHaveBeenCalledTimes(1); - }); - - it("should create and enable OpenAI Codex plugin by default", () => { - const plugin = new BraintrustPlugin(); - plugin.enable(); - - expect(OpenAICodexPlugin).toHaveBeenCalledTimes(1); - const mockInstance = vi.mocked(OpenAICodexPlugin).mock.results[0].value; - expect(mockInstance.enable).toHaveBeenCalledTimes(1); - }); - - it("should create and enable AI SDK plugin by default", () => { - const plugin = new BraintrustPlugin(); - plugin.enable(); - - expect(AISDKPlugin).toHaveBeenCalledTimes(1); - const mockInstance = vi.mocked(AISDKPlugin).mock.results[0].value; - expect(mockInstance.enable).toHaveBeenCalledTimes(1); - }); - - it("should create and enable Claude Agent SDK plugin by default", () => { - const plugin = new BraintrustPlugin(); - plugin.enable(); - - expect(ClaudeAgentSDKPlugin).toHaveBeenCalledTimes(1); - const mockInstance = - vi.mocked(ClaudeAgentSDKPlugin).mock.results[0].value; - expect(mockInstance.enable).toHaveBeenCalledTimes(1); - }); - - it("should create and enable Cloudflare Think plugin by default", () => { - const plugin = new BraintrustPlugin(); - plugin.enable(); - - expect(CloudflareThinkPlugin).toHaveBeenCalledTimes(1); - const mockInstance = vi.mocked(CloudflareThinkPlugin).mock.results[0] - .value; - expect(mockInstance.enable).toHaveBeenCalledTimes(1); - }); - - it("should create and enable OpenAI Agents plugin by default", () => { - const plugin = new BraintrustPlugin(); - plugin.enable(); - - expect(OpenAIAgentsPlugin).toHaveBeenCalledTimes(1); - const mockInstance = vi.mocked(OpenAIAgentsPlugin).mock.results[0].value; - expect(mockInstance.enable).toHaveBeenCalledTimes(1); - }); - - it("should create and enable Google GenAI plugin by default", () => { - const plugin = new BraintrustPlugin(); - plugin.enable(); - - expect(GoogleGenAIPlugin).toHaveBeenCalledTimes(1); - const mockInstance = vi.mocked(GoogleGenAIPlugin).mock.results[0].value; - expect(mockInstance.enable).toHaveBeenCalledTimes(1); - }); - - it("should create and enable HuggingFace plugin by default", () => { - const plugin = new BraintrustPlugin(); - plugin.enable(); - - expect(HuggingFacePlugin).toHaveBeenCalledTimes(1); - const mockInstance = vi.mocked(HuggingFacePlugin).mock.results[0].value; - expect(mockInstance.enable).toHaveBeenCalledTimes(1); - expect(HuggingFaceTransformersPlugin).toHaveBeenCalledTimes(1); - const transformersMockInstance = vi.mocked(HuggingFaceTransformersPlugin) - .mock.results[0].value; - expect(transformersMockInstance.enable).toHaveBeenCalledTimes(1); - }); - - it("should create and enable OpenRouter plugin by default", () => { - const plugin = new BraintrustPlugin(); - plugin.enable(); - - expect(OpenRouterPlugin).toHaveBeenCalledTimes(1); - expect(OpenRouterAgentPlugin).toHaveBeenCalledTimes(1); - const mockInstance = vi.mocked(OpenRouterPlugin).mock.results[0].value; - expect(mockInstance.enable).toHaveBeenCalledTimes(1); - }); - - it("should create and enable OpenRouter Agent plugin by default", () => { - const plugin = new BraintrustPlugin(); - plugin.enable(); - - expect(OpenRouterAgentPlugin).toHaveBeenCalledTimes(1); - const mockInstance = vi.mocked(OpenRouterAgentPlugin).mock.results[0] - .value; - expect(mockInstance.enable).toHaveBeenCalledTimes(1); - }); - - it("should create and enable Mistral plugin by default", () => { - const plugin = new BraintrustPlugin(); - plugin.enable(); - - expect(MistralPlugin).toHaveBeenCalledTimes(1); - const mockInstance = vi.mocked(MistralPlugin).mock.results[0].value; - expect(mockInstance.enable).toHaveBeenCalledTimes(1); - }); - - it("should create and enable Cohere plugin by default", () => { - const plugin = new BraintrustPlugin(); - plugin.enable(); - - expect(CoherePlugin).toHaveBeenCalledTimes(1); - const mockInstance = vi.mocked(CoherePlugin).mock.results[0].value; - expect(mockInstance.enable).toHaveBeenCalledTimes(1); - }); - - it("should create and enable Voyage AI plugin by default", () => { - const plugin = new BraintrustPlugin(); - plugin.enable(); - - expect(VoyageAIPlugin).toHaveBeenCalledTimes(1); - const mockInstance = vi.mocked(VoyageAIPlugin).mock.results[0].value; - expect(mockInstance.enable).toHaveBeenCalledTimes(1); - }); - - it("should create and enable Groq plugin by default", () => { - const plugin = new BraintrustPlugin(); - plugin.enable(); - - expect(GroqPlugin).toHaveBeenCalledTimes(1); - const mockInstance = vi.mocked(GroqPlugin).mock.results[0].value; - expect(mockInstance.enable).toHaveBeenCalledTimes(1); - }); - - it("should create and enable GitHubCopilot plugin by default", () => { - const plugin = new BraintrustPlugin(); - plugin.enable(); - - expect(GitHubCopilotPlugin).toHaveBeenCalledTimes(1); - const mockInstance = vi.mocked(GitHubCopilotPlugin).mock.results[0].value; - expect(mockInstance.enable).toHaveBeenCalledTimes(1); - }); - - it("should create and enable LangChain plugin by default", () => { - const plugin = new BraintrustPlugin(); - plugin.enable(); - - expect(LangChainPlugin).toHaveBeenCalledTimes(1); - const mockInstance = vi.mocked(LangChainPlugin).mock.results[0].value; - expect(mockInstance.enable).toHaveBeenCalledTimes(1); - }); - - it("should create and enable Strands Agent SDK plugin by default", () => { - const plugin = new BraintrustPlugin(); - plugin.enable(); - - expect(StrandsAgentSDKPlugin).toHaveBeenCalledTimes(1); - const mockInstance = vi.mocked(StrandsAgentSDKPlugin).mock.results[0] - .value; - expect(mockInstance.enable).toHaveBeenCalledTimes(1); - }); - - it("should create and enable Cloudflare AI Chat plugin by default", () => { - const plugin = new BraintrustPlugin(); - plugin.enable(); - - expect(CloudflareAIChatPlugin).toHaveBeenCalledTimes(1); - const mockInstance = vi.mocked(CloudflareAIChatPlugin).mock.results[0] - .value; - expect(mockInstance.enable).toHaveBeenCalledTimes(1); - }); - - it("should create and enable Cloudflare Agents plugin by default", () => { - const plugin = new BraintrustPlugin(); - plugin.enable(); - - expect(CloudflareAgentsPlugin).toHaveBeenCalledTimes(1); - const mockInstance = vi.mocked(CloudflareAgentsPlugin).mock.results[0] - .value; - expect(mockInstance.enable).toHaveBeenCalledTimes(1); - }); - - it("should create all plugins when enabled with no config", () => { - const plugin = new BraintrustPlugin(); - plugin.enable(); - - expect(OpenAIPlugin).toHaveBeenCalledTimes(1); - expect(OpenAICodexPlugin).toHaveBeenCalledTimes(1); - expect(AnthropicPlugin).toHaveBeenCalledTimes(1); - expect(AISDKPlugin).toHaveBeenCalledTimes(1); - expect(ClaudeAgentSDKPlugin).toHaveBeenCalledTimes(1); - expect(OpenAIAgentsPlugin).toHaveBeenCalledTimes(1); - expect(GoogleGenAIPlugin).toHaveBeenCalledTimes(1); - expect(HuggingFacePlugin).toHaveBeenCalledTimes(1); - expect(OpenRouterPlugin).toHaveBeenCalledTimes(1); - expect(OpenRouterAgentPlugin).toHaveBeenCalledTimes(1); - expect(MistralPlugin).toHaveBeenCalledTimes(1); - expect(CoherePlugin).toHaveBeenCalledTimes(1); - expect(GroqPlugin).toHaveBeenCalledTimes(1); - expect(GitHubCopilotPlugin).toHaveBeenCalledTimes(1); - expect(StrandsAgentSDKPlugin).toHaveBeenCalledTimes(1); - expect(LangChainPlugin).toHaveBeenCalledTimes(1); - }); - - it("should create all plugins when enabled with empty config", () => { - const plugin = new BraintrustPlugin({}); - plugin.enable(); - - expect(OpenAIPlugin).toHaveBeenCalledTimes(1); - expect(OpenAICodexPlugin).toHaveBeenCalledTimes(1); - expect(AnthropicPlugin).toHaveBeenCalledTimes(1); - expect(AISDKPlugin).toHaveBeenCalledTimes(1); - expect(ClaudeAgentSDKPlugin).toHaveBeenCalledTimes(1); - expect(OpenAIAgentsPlugin).toHaveBeenCalledTimes(1); - expect(GoogleGenAIPlugin).toHaveBeenCalledTimes(1); - expect(HuggingFacePlugin).toHaveBeenCalledTimes(1); - expect(OpenRouterPlugin).toHaveBeenCalledTimes(1); - expect(OpenRouterAgentPlugin).toHaveBeenCalledTimes(1); - expect(MistralPlugin).toHaveBeenCalledTimes(1); - expect(CoherePlugin).toHaveBeenCalledTimes(1); - expect(GroqPlugin).toHaveBeenCalledTimes(1); - expect(GitHubCopilotPlugin).toHaveBeenCalledTimes(1); - expect(StrandsAgentSDKPlugin).toHaveBeenCalledTimes(1); - expect(LangChainPlugin).toHaveBeenCalledTimes(1); - }); - - it("should create all plugins when enabled with empty integrations config", () => { - const plugin = new BraintrustPlugin({ integrations: {} }); - plugin.enable(); - - expect(OpenAIPlugin).toHaveBeenCalledTimes(1); - expect(OpenAICodexPlugin).toHaveBeenCalledTimes(1); - expect(AnthropicPlugin).toHaveBeenCalledTimes(1); - expect(AISDKPlugin).toHaveBeenCalledTimes(1); - expect(ClaudeAgentSDKPlugin).toHaveBeenCalledTimes(1); - expect(OpenAIAgentsPlugin).toHaveBeenCalledTimes(1); - expect(GoogleGenAIPlugin).toHaveBeenCalledTimes(1); - expect(HuggingFacePlugin).toHaveBeenCalledTimes(1); - expect(OpenRouterPlugin).toHaveBeenCalledTimes(1); - expect(OpenRouterAgentPlugin).toHaveBeenCalledTimes(1); - expect(MistralPlugin).toHaveBeenCalledTimes(1); - expect(CoherePlugin).toHaveBeenCalledTimes(1); - expect(GroqPlugin).toHaveBeenCalledTimes(1); - expect(StrandsAgentSDKPlugin).toHaveBeenCalledTimes(1); - expect(LangChainPlugin).toHaveBeenCalledTimes(1); - }); - }); - - describe("config-based disabling", () => { - it("should not create OpenAI plugin when openai: false", () => { - const plugin = new BraintrustPlugin({ - integrations: { openai: false }, - }); - plugin.enable(); - - expect(OpenAIPlugin).not.toHaveBeenCalled(); - // Other plugins should still be created - expect(AnthropicPlugin).toHaveBeenCalledTimes(1); - expect(AISDKPlugin).toHaveBeenCalledTimes(1); - expect(ClaudeAgentSDKPlugin).toHaveBeenCalledTimes(1); - expect(OpenAIAgentsPlugin).toHaveBeenCalledTimes(1); - expect(GoogleGenAIPlugin).toHaveBeenCalledTimes(1); - expect(HuggingFacePlugin).toHaveBeenCalledTimes(1); - expect(OpenRouterPlugin).toHaveBeenCalledTimes(1); - expect(OpenRouterAgentPlugin).toHaveBeenCalledTimes(1); - }); - - it("should not create Anthropic plugin when anthropic: false", () => { - const plugin = new BraintrustPlugin({ - integrations: { anthropic: false }, - }); - plugin.enable(); - - expect(AnthropicPlugin).not.toHaveBeenCalled(); - // Other plugins should still be created - expect(OpenAIPlugin).toHaveBeenCalledTimes(1); - expect(OpenAICodexPlugin).toHaveBeenCalledTimes(1); - expect(AISDKPlugin).toHaveBeenCalledTimes(1); - expect(ClaudeAgentSDKPlugin).toHaveBeenCalledTimes(1); - expect(OpenAIAgentsPlugin).toHaveBeenCalledTimes(1); - expect(GoogleGenAIPlugin).toHaveBeenCalledTimes(1); - expect(HuggingFacePlugin).toHaveBeenCalledTimes(1); - expect(OpenRouterPlugin).toHaveBeenCalledTimes(1); - expect(OpenRouterAgentPlugin).toHaveBeenCalledTimes(1); - }); - - it("should not create OpenAI Codex plugin when openaiCodexSDK: false", () => { - const plugin = new BraintrustPlugin({ - integrations: { openaiCodexSDK: false }, - }); - plugin.enable(); - - expect(OpenAICodexPlugin).not.toHaveBeenCalled(); - expect(OpenAIPlugin).toHaveBeenCalledTimes(1); - expect(AnthropicPlugin).toHaveBeenCalledTimes(1); - expect(AISDKPlugin).toHaveBeenCalledTimes(1); - }); - - it("should not create AI SDK plugin when aisdk: false", () => { - const plugin = new BraintrustPlugin({ - integrations: { aisdk: false }, - }); - plugin.enable(); - - expect(AISDKPlugin).not.toHaveBeenCalled(); - // Other plugins should still be created - expect(OpenAIPlugin).toHaveBeenCalledTimes(1); - expect(AnthropicPlugin).toHaveBeenCalledTimes(1); - expect(ClaudeAgentSDKPlugin).toHaveBeenCalledTimes(1); - expect(GoogleGenAIPlugin).toHaveBeenCalledTimes(1); - expect(HuggingFacePlugin).toHaveBeenCalledTimes(1); - expect(OpenRouterPlugin).toHaveBeenCalledTimes(1); - expect(OpenRouterAgentPlugin).toHaveBeenCalledTimes(1); - expect(MistralPlugin).toHaveBeenCalledTimes(1); - }); - - it("should not create Claude Agent SDK plugin when claudeAgentSDK: false", () => { - const plugin = new BraintrustPlugin({ - integrations: { claudeAgentSDK: false }, - }); - plugin.enable(); - - expect(ClaudeAgentSDKPlugin).not.toHaveBeenCalled(); - // Other plugins should still be created - expect(OpenAIPlugin).toHaveBeenCalledTimes(1); - expect(AnthropicPlugin).toHaveBeenCalledTimes(1); - expect(AISDKPlugin).toHaveBeenCalledTimes(1); - expect(GoogleGenAIPlugin).toHaveBeenCalledTimes(1); - expect(OpenAIAgentsPlugin).toHaveBeenCalledTimes(1); - expect(HuggingFacePlugin).toHaveBeenCalledTimes(1); - expect(OpenRouterPlugin).toHaveBeenCalledTimes(1); - expect(OpenRouterAgentPlugin).toHaveBeenCalledTimes(1); - expect(MistralPlugin).toHaveBeenCalledTimes(1); - }); - - it("should not create Cloudflare Think plugin when cloudflareThink: false", () => { - const plugin = new BraintrustPlugin({ - integrations: { cloudflareThink: false }, - }); - plugin.enable(); - - expect(CloudflareThinkPlugin).not.toHaveBeenCalled(); - expect(AISDKPlugin).toHaveBeenCalledTimes(1); - }); - - it("should not create OpenAI Agents plugin when openAIAgents: false", () => { - const plugin = new BraintrustPlugin({ - integrations: { openAIAgents: false }, - }); - plugin.enable(); - - expect(OpenAIAgentsPlugin).not.toHaveBeenCalled(); - expect(OpenAIPlugin).toHaveBeenCalledTimes(1); - expect(AnthropicPlugin).toHaveBeenCalledTimes(1); - expect(AISDKPlugin).toHaveBeenCalledTimes(1); - expect(ClaudeAgentSDKPlugin).toHaveBeenCalledTimes(1); - expect(GoogleGenAIPlugin).toHaveBeenCalledTimes(1); - expect(HuggingFacePlugin).toHaveBeenCalledTimes(1); - expect(OpenRouterPlugin).toHaveBeenCalledTimes(1); - expect(OpenRouterAgentPlugin).toHaveBeenCalledTimes(1); - expect(MistralPlugin).toHaveBeenCalledTimes(1); - }); - - it("should not create HuggingFace plugin when huggingface: false", () => { - const plugin = new BraintrustPlugin({ - integrations: { huggingface: false }, - }); - plugin.enable(); - - expect(HuggingFacePlugin).not.toHaveBeenCalled(); - expect(HuggingFaceTransformersPlugin).not.toHaveBeenCalled(); - expect(OpenAIPlugin).toHaveBeenCalledTimes(1); - expect(AnthropicPlugin).toHaveBeenCalledTimes(1); - expect(AISDKPlugin).toHaveBeenCalledTimes(1); - expect(ClaudeAgentSDKPlugin).toHaveBeenCalledTimes(1); - expect(GoogleGenAIPlugin).toHaveBeenCalledTimes(1); - expect(OpenRouterPlugin).toHaveBeenCalledTimes(1); - expect(OpenRouterAgentPlugin).toHaveBeenCalledTimes(1); - expect(MistralPlugin).toHaveBeenCalledTimes(1); - }); - - it("should not create Google GenAI plugin when googleGenAI: false", () => { - const plugin = new BraintrustPlugin({ - integrations: { googleGenAI: false }, - }); - plugin.enable(); - - expect(GoogleGenAIPlugin).not.toHaveBeenCalled(); - // Other plugins should still be created - expect(OpenAIPlugin).toHaveBeenCalledTimes(1); - expect(AnthropicPlugin).toHaveBeenCalledTimes(1); - expect(AISDKPlugin).toHaveBeenCalledTimes(1); - expect(ClaudeAgentSDKPlugin).toHaveBeenCalledTimes(1); - expect(OpenRouterPlugin).toHaveBeenCalledTimes(1); - expect(OpenRouterAgentPlugin).toHaveBeenCalledTimes(1); - expect(MistralPlugin).toHaveBeenCalledTimes(1); - }); - - it("should not create OpenRouter plugin when openrouter: false", () => { - const plugin = new BraintrustPlugin({ - integrations: { openrouter: false }, - }); - plugin.enable(); - - expect(OpenRouterPlugin).not.toHaveBeenCalled(); - expect(OpenRouterAgentPlugin).toHaveBeenCalledTimes(1); - expect(OpenAIPlugin).toHaveBeenCalledTimes(1); - expect(AnthropicPlugin).toHaveBeenCalledTimes(1); - expect(AISDKPlugin).toHaveBeenCalledTimes(1); - expect(ClaudeAgentSDKPlugin).toHaveBeenCalledTimes(1); - expect(GoogleGenAIPlugin).toHaveBeenCalledTimes(1); - expect(HuggingFacePlugin).toHaveBeenCalledTimes(1); - expect(MistralPlugin).toHaveBeenCalledTimes(1); - }); - - it("should not create Mistral plugin when mistral: false", () => { - const plugin = new BraintrustPlugin({ - integrations: { mistral: false }, - }); - plugin.enable(); - - expect(MistralPlugin).not.toHaveBeenCalled(); - expect(OpenAIPlugin).toHaveBeenCalledTimes(1); - expect(AnthropicPlugin).toHaveBeenCalledTimes(1); - expect(AISDKPlugin).toHaveBeenCalledTimes(1); - expect(ClaudeAgentSDKPlugin).toHaveBeenCalledTimes(1); - expect(GoogleGenAIPlugin).toHaveBeenCalledTimes(1); - expect(HuggingFacePlugin).toHaveBeenCalledTimes(1); - expect(OpenRouterPlugin).toHaveBeenCalledTimes(1); - }); - - it("should not create Ollama plugin when ollama: false", () => { - const plugin = new BraintrustPlugin({ - integrations: { ollama: false }, - }); - plugin.enable(); - - expect(OllamaPlugin).not.toHaveBeenCalled(); - expect(OpenAIPlugin).toHaveBeenCalledTimes(1); - expect(AnthropicPlugin).toHaveBeenCalledTimes(1); - expect(MistralPlugin).toHaveBeenCalledTimes(1); - }); - - it("should not create Cohere plugin when cohere: false", () => { - const plugin = new BraintrustPlugin({ - integrations: { cohere: false }, - }); - plugin.enable(); - - expect(CoherePlugin).not.toHaveBeenCalled(); - expect(OpenAIPlugin).toHaveBeenCalledTimes(1); - expect(AnthropicPlugin).toHaveBeenCalledTimes(1); - expect(AISDKPlugin).toHaveBeenCalledTimes(1); - expect(ClaudeAgentSDKPlugin).toHaveBeenCalledTimes(1); - expect(GoogleGenAIPlugin).toHaveBeenCalledTimes(1); - expect(OpenRouterPlugin).toHaveBeenCalledTimes(1); - expect(MistralPlugin).toHaveBeenCalledTimes(1); - }); - - it("should not create Groq plugin when groq: false", () => { - const plugin = new BraintrustPlugin({ - integrations: { groq: false }, - }); - plugin.enable(); - - expect(GroqPlugin).not.toHaveBeenCalled(); - expect(OpenAIPlugin).toHaveBeenCalledTimes(1); - expect(AnthropicPlugin).toHaveBeenCalledTimes(1); - expect(AISDKPlugin).toHaveBeenCalledTimes(1); - expect(ClaudeAgentSDKPlugin).toHaveBeenCalledTimes(1); - expect(GoogleGenAIPlugin).toHaveBeenCalledTimes(1); - expect(OpenRouterPlugin).toHaveBeenCalledTimes(1); - expect(MistralPlugin).toHaveBeenCalledTimes(1); - }); - - it("should not create GitHubCopilot plugin when gitHubCopilot: false", () => { - const plugin = new BraintrustPlugin({ - integrations: { gitHubCopilot: false }, - }); - plugin.enable(); - - expect(GitHubCopilotPlugin).not.toHaveBeenCalled(); - expect(OpenAIPlugin).toHaveBeenCalledTimes(1); - expect(GroqPlugin).toHaveBeenCalledTimes(1); - }); - - it("should not create LangChain plugin when langchain: false", () => { - const plugin = new BraintrustPlugin({ - integrations: { langchain: false }, - }); - plugin.enable(); - - expect(LangChainPlugin).not.toHaveBeenCalled(); - expect(OpenAIPlugin).toHaveBeenCalledTimes(1); - expect(AnthropicPlugin).toHaveBeenCalledTimes(1); - }); - - it("should create LangSmith with LangChain deduplication by default", () => { - const plugin = new BraintrustPlugin(); - plugin.enable(); - - expect(LangSmithPlugin).toHaveBeenCalledWith({ - skipLangChainRuns: true, - }); - const mockInstance = vi.mocked(LangSmithPlugin).mock.results[0].value; - expect(mockInstance.enable).toHaveBeenCalledTimes(1); - }); - - it("should disable LangSmith or let it capture LangChain runs", () => { - const disabled = new BraintrustPlugin({ - integrations: { langsmith: false }, - }); - disabled.enable(); - expect(LangSmithPlugin).not.toHaveBeenCalled(); - - const withoutLangChain = new BraintrustPlugin({ - integrations: { langchain: false }, - }); - withoutLangChain.enable(); - expect(LangSmithPlugin).toHaveBeenCalledWith({ - skipLangChainRuns: false, - }); - }); - - it("should not create OpenRouter Agent plugin when openrouterAgent: false", () => { - const plugin = new BraintrustPlugin({ - integrations: { openrouterAgent: false }, - }); - plugin.enable(); - - expect(OpenRouterPlugin).toHaveBeenCalledTimes(1); - expect(OpenRouterAgentPlugin).not.toHaveBeenCalled(); - }); - - it("should not create any plugins when all are disabled", () => { - const plugin = new BraintrustPlugin({ - integrations: { - openai: false, - openaiCodexSDK: false, - anthropic: false, - aisdk: false, - claudeAgentSDK: false, - cloudflareThink: false, - openAIAgents: false, - googleGenAI: false, - huggingface: false, - openrouter: false, - openrouterAgent: false, - mistral: false, - ollama: false, - cohere: false, - groq: false, - gitHubCopilot: false, - langchain: false, - langsmith: false, - piCodingAgent: false, - strandsAgentSDK: false, - cloudflareAIChat: false, - cloudflareAgents: false, - }, - }); - plugin.enable(); - - expect(OpenAIPlugin).not.toHaveBeenCalled(); - expect(OpenAICodexPlugin).not.toHaveBeenCalled(); - expect(AnthropicPlugin).not.toHaveBeenCalled(); - expect(AISDKPlugin).not.toHaveBeenCalled(); - expect(ClaudeAgentSDKPlugin).not.toHaveBeenCalled(); - expect(CloudflareThinkPlugin).not.toHaveBeenCalled(); - expect(OpenAIAgentsPlugin).not.toHaveBeenCalled(); - expect(GoogleGenAIPlugin).not.toHaveBeenCalled(); - expect(HuggingFacePlugin).not.toHaveBeenCalled(); - expect(OpenRouterPlugin).not.toHaveBeenCalled(); - expect(OpenRouterAgentPlugin).not.toHaveBeenCalled(); - expect(MistralPlugin).not.toHaveBeenCalled(); - expect(OllamaPlugin).not.toHaveBeenCalled(); - expect(CoherePlugin).not.toHaveBeenCalled(); - expect(GroqPlugin).not.toHaveBeenCalled(); - expect(GitHubCopilotPlugin).not.toHaveBeenCalled(); - expect(LangChainPlugin).not.toHaveBeenCalled(); - expect(LangSmithPlugin).not.toHaveBeenCalled(); - expect(PiCodingAgentPlugin).not.toHaveBeenCalled(); - expect(StrandsAgentSDKPlugin).not.toHaveBeenCalled(); - expect(CloudflareAIChatPlugin).not.toHaveBeenCalled(); - expect(CloudflareAgentsPlugin).not.toHaveBeenCalled(); - }); - - it("should not create Pi Coding Agent plugin when piCodingAgent: false", () => { - const plugin = new BraintrustPlugin({ - integrations: { piCodingAgent: false }, - }); - plugin.enable(); - - expect(PiCodingAgentPlugin).not.toHaveBeenCalled(); - expect(OpenAIPlugin).toHaveBeenCalledTimes(1); - expect(AnthropicPlugin).toHaveBeenCalledTimes(1); - }); - - it("should not create Strands Agent SDK plugin when strandsAgentSDK: false", () => { - const plugin = new BraintrustPlugin({ - integrations: { strandsAgentSDK: false }, - }); - plugin.enable(); - - expect(StrandsAgentSDKPlugin).not.toHaveBeenCalled(); - expect(OpenAIPlugin).toHaveBeenCalledTimes(1); - }); - - it("should not create Cloudflare AI Chat plugin when cloudflareAIChat: false", () => { - const plugin = new BraintrustPlugin({ - integrations: { cloudflareAIChat: false }, - }); - plugin.enable(); - - expect(CloudflareAIChatPlugin).not.toHaveBeenCalled(); - expect(OpenAIPlugin).toHaveBeenCalledTimes(1); - }); - - it("should not create Cloudflare Agents plugin when cloudflareAgents: false", () => { - const plugin = new BraintrustPlugin({ - integrations: { cloudflareAgents: false }, - }); - plugin.enable(); - - expect(CloudflareAgentsPlugin).not.toHaveBeenCalled(); - expect(OpenAIPlugin).toHaveBeenCalledTimes(1); - }); - - it("should allow selective enabling of plugins", () => { - const plugin = new BraintrustPlugin({ - integrations: { - openai: true, - anthropic: false, - aisdk: false, - claudeAgentSDK: true, - openAIAgents: true, - googleGenAI: false, - huggingface: true, - openrouter: true, - mistral: false, - }, - }); - plugin.enable(); - - expect(OpenAIPlugin).toHaveBeenCalledTimes(1); - expect(ClaudeAgentSDKPlugin).toHaveBeenCalledTimes(1); - expect(OpenAIAgentsPlugin).toHaveBeenCalledTimes(1); - expect(HuggingFacePlugin).toHaveBeenCalledTimes(1); - expect(OpenRouterPlugin).toHaveBeenCalledTimes(1); - expect(OpenRouterAgentPlugin).toHaveBeenCalledTimes(1); - expect(AnthropicPlugin).not.toHaveBeenCalled(); - expect(AISDKPlugin).not.toHaveBeenCalled(); - expect(GoogleGenAIPlugin).not.toHaveBeenCalled(); - expect(MistralPlugin).not.toHaveBeenCalled(); - }); - }); - - describe("backward compatibility (legacy config keys)", () => { - it("should not create AI SDK plugin when vercel: false (legacy)", () => { - const plugin = new BraintrustPlugin({ - integrations: { vercel: false }, - }); - plugin.enable(); - - expect(AISDKPlugin).not.toHaveBeenCalled(); - // Other plugins should still be created - expect(OpenAIPlugin).toHaveBeenCalledTimes(1); - expect(AnthropicPlugin).toHaveBeenCalledTimes(1); - expect(ClaudeAgentSDKPlugin).toHaveBeenCalledTimes(1); - expect(OpenAIAgentsPlugin).toHaveBeenCalledTimes(1); - expect(GoogleGenAIPlugin).toHaveBeenCalledTimes(1); - expect(HuggingFacePlugin).toHaveBeenCalledTimes(1); - expect(OpenRouterPlugin).toHaveBeenCalledTimes(1); - expect(MistralPlugin).toHaveBeenCalledTimes(1); - }); - - it("should not create Google GenAI plugin when google: false (legacy)", () => { - const plugin = new BraintrustPlugin({ - integrations: { google: false }, - }); - plugin.enable(); - - expect(GoogleGenAIPlugin).not.toHaveBeenCalled(); - // Other plugins should still be created - expect(OpenAIPlugin).toHaveBeenCalledTimes(1); - expect(AnthropicPlugin).toHaveBeenCalledTimes(1); - expect(AISDKPlugin).toHaveBeenCalledTimes(1); - expect(ClaudeAgentSDKPlugin).toHaveBeenCalledTimes(1); - expect(HuggingFacePlugin).toHaveBeenCalledTimes(1); - expect(OpenRouterPlugin).toHaveBeenCalledTimes(1); - expect(MistralPlugin).toHaveBeenCalledTimes(1); - }); - - it("should not create LangChain plugin when langgraph: false (alias)", () => { - const plugin = new BraintrustPlugin({ - integrations: { langgraph: false }, - }); - plugin.enable(); - - expect(LangChainPlugin).not.toHaveBeenCalled(); - expect(LangSmithPlugin).toHaveBeenCalledWith({ - skipLangChainRuns: true, - }); - }); - - it("should not create AI SDK plugin when both aisdk and vercel are false", () => { - const plugin = new BraintrustPlugin({ - integrations: { aisdk: false, vercel: false }, - }); - plugin.enable(); - - expect(AISDKPlugin).not.toHaveBeenCalled(); - }); - - it("should not create Google GenAI plugin when both googleGenAI and google are false", () => { - const plugin = new BraintrustPlugin({ - integrations: { googleGenAI: false, google: false }, - }); - plugin.enable(); - - expect(GoogleGenAIPlugin).not.toHaveBeenCalled(); - }); - - it("should not create AI SDK plugin when aisdk is true but vercel is false", () => { - const plugin = new BraintrustPlugin({ - integrations: { aisdk: true, vercel: false }, - }); - plugin.enable(); - - expect(AISDKPlugin).not.toHaveBeenCalled(); - }); - - it("should not create Google GenAI plugin when googleGenAI is true but google is false", () => { - const plugin = new BraintrustPlugin({ - integrations: { googleGenAI: true, google: false }, - }); - plugin.enable(); - - expect(GoogleGenAIPlugin).not.toHaveBeenCalled(); - }); - - it("should create AI SDK plugin when vercel is true and aisdk is not set", () => { - const plugin = new BraintrustPlugin({ - integrations: { vercel: true }, - }); - plugin.enable(); - - expect(AISDKPlugin).toHaveBeenCalledTimes(1); - }); - - it("should create Google GenAI plugin when google is true and googleGenAI is not set", () => { - const plugin = new BraintrustPlugin({ - integrations: { google: true }, - }); - plugin.enable(); - - expect(GoogleGenAIPlugin).toHaveBeenCalledTimes(1); - }); - }); - - describe("enable/disable lifecycle", () => { - it("should enable all sub-plugins when enabled", () => { - const plugin = new BraintrustPlugin(); - plugin.enable(); - - const openaiMock = vi.mocked(OpenAIPlugin).mock.results[0].value; - const openAICodexMock = - vi.mocked(OpenAICodexPlugin).mock.results[0].value; - const anthropicMock = vi.mocked(AnthropicPlugin).mock.results[0].value; - const aiSDKMock = vi.mocked(AISDKPlugin).mock.results[0].value; - const claudeAgentSDKMock = - vi.mocked(ClaudeAgentSDKPlugin).mock.results[0].value; - const openAIAgentsMock = - vi.mocked(OpenAIAgentsPlugin).mock.results[0].value; - const googleGenAIMock = - vi.mocked(GoogleGenAIPlugin).mock.results[0].value; - const huggingFaceMock = - vi.mocked(HuggingFacePlugin).mock.results[0].value; - const huggingFaceTransformersMock = vi.mocked( - HuggingFaceTransformersPlugin, - ).mock.results[0].value; - const openRouterMock = vi.mocked(OpenRouterPlugin).mock.results[0].value; - const openRouterAgentMock = vi.mocked(OpenRouterAgentPlugin).mock - .results[0].value; - const mistralMock = vi.mocked(MistralPlugin).mock.results[0].value; - const ollamaMock = vi.mocked(OllamaPlugin).mock.results[0].value; - const cohereMock = vi.mocked(CoherePlugin).mock.results[0].value; - const groqMock = vi.mocked(GroqPlugin).mock.results[0].value; - const piCodingAgentMock = - vi.mocked(PiCodingAgentPlugin).mock.results[0].value; - const strandsAgentSDKMock = vi.mocked(StrandsAgentSDKPlugin).mock - .results[0].value; - const cloudflareAgentsMock = vi.mocked(CloudflareAgentsPlugin).mock - .results[0].value; - const langChainMock = vi.mocked(LangChainPlugin).mock.results[0].value; - - expect(openaiMock.enable).toHaveBeenCalledTimes(1); - expect(openAICodexMock.enable).toHaveBeenCalledTimes(1); - expect(anthropicMock.enable).toHaveBeenCalledTimes(1); - expect(aiSDKMock.enable).toHaveBeenCalledTimes(1); - expect(claudeAgentSDKMock.enable).toHaveBeenCalledTimes(1); - expect(openAIAgentsMock.enable).toHaveBeenCalledTimes(1); - expect(googleGenAIMock.enable).toHaveBeenCalledTimes(1); - expect(huggingFaceMock.enable).toHaveBeenCalledTimes(1); - expect(huggingFaceTransformersMock.enable).toHaveBeenCalledTimes(1); - expect(openRouterMock.enable).toHaveBeenCalledTimes(1); - expect(openRouterAgentMock.enable).toHaveBeenCalledTimes(1); - expect(mistralMock.enable).toHaveBeenCalledTimes(1); - expect(ollamaMock.enable).toHaveBeenCalledTimes(1); - expect(cohereMock.enable).toHaveBeenCalledTimes(1); - expect(groqMock.enable).toHaveBeenCalledTimes(1); - expect(piCodingAgentMock.enable).toHaveBeenCalledTimes(1); - expect(strandsAgentSDKMock.enable).toHaveBeenCalledTimes(1); - expect(cloudflareAgentsMock.enable).toHaveBeenCalledTimes(1); - expect(langChainMock.enable).toHaveBeenCalledTimes(1); - }); - - it("should disable and nullify all sub-plugins when disabled", () => { - const plugin = new BraintrustPlugin(); - plugin.enable(); - - const openaiMock = vi.mocked(OpenAIPlugin).mock.results[0].value; - const openAICodexMock = - vi.mocked(OpenAICodexPlugin).mock.results[0].value; - const anthropicMock = vi.mocked(AnthropicPlugin).mock.results[0].value; - const aiSDKMock = vi.mocked(AISDKPlugin).mock.results[0].value; - const claudeAgentSDKMock = - vi.mocked(ClaudeAgentSDKPlugin).mock.results[0].value; - const openAIAgentsMock = - vi.mocked(OpenAIAgentsPlugin).mock.results[0].value; - const googleGenAIMock = - vi.mocked(GoogleGenAIPlugin).mock.results[0].value; - const huggingFaceMock = - vi.mocked(HuggingFacePlugin).mock.results[0].value; - const huggingFaceTransformersMock = vi.mocked( - HuggingFaceTransformersPlugin, - ).mock.results[0].value; - const openRouterMock = vi.mocked(OpenRouterPlugin).mock.results[0].value; - const openRouterAgentMock = vi.mocked(OpenRouterAgentPlugin).mock - .results[0].value; - const mistralMock = vi.mocked(MistralPlugin).mock.results[0].value; - const ollamaMock = vi.mocked(OllamaPlugin).mock.results[0].value; - const cohereMock = vi.mocked(CoherePlugin).mock.results[0].value; - const groqMock = vi.mocked(GroqPlugin).mock.results[0].value; - const piCodingAgentMock = - vi.mocked(PiCodingAgentPlugin).mock.results[0].value; - const strandsAgentSDKMock = vi.mocked(StrandsAgentSDKPlugin).mock - .results[0].value; - const cloudflareAgentsMock = vi.mocked(CloudflareAgentsPlugin).mock - .results[0].value; - const langChainMock = vi.mocked(LangChainPlugin).mock.results[0].value; - - plugin.disable(); - - expect(openaiMock.disable).toHaveBeenCalledTimes(1); - expect(openAICodexMock.disable).toHaveBeenCalledTimes(1); - expect(anthropicMock.disable).toHaveBeenCalledTimes(1); - expect(aiSDKMock.disable).toHaveBeenCalledTimes(1); - expect(claudeAgentSDKMock.disable).toHaveBeenCalledTimes(1); - expect(openAIAgentsMock.disable).toHaveBeenCalledTimes(1); - expect(googleGenAIMock.disable).toHaveBeenCalledTimes(1); - expect(huggingFaceMock.disable).toHaveBeenCalledTimes(1); - expect(huggingFaceTransformersMock.disable).toHaveBeenCalledTimes(1); - expect(openRouterMock.disable).toHaveBeenCalledTimes(1); - expect(openRouterAgentMock.disable).toHaveBeenCalledTimes(1); - expect(mistralMock.disable).toHaveBeenCalledTimes(1); - expect(ollamaMock.disable).toHaveBeenCalledTimes(1); - expect(cohereMock.disable).toHaveBeenCalledTimes(1); - expect(groqMock.disable).toHaveBeenCalledTimes(1); - expect(piCodingAgentMock.disable).toHaveBeenCalledTimes(1); - expect(strandsAgentSDKMock.disable).toHaveBeenCalledTimes(1); - expect(cloudflareAgentsMock.disable).toHaveBeenCalledTimes(1); - expect(langChainMock.disable).toHaveBeenCalledTimes(1); - }); - - it("should be idempotent on multiple enable calls", () => { - const plugin = new BraintrustPlugin(); - plugin.enable(); - plugin.enable(); - plugin.enable(); - - // Should only create and enable plugins once - expect(OpenAIPlugin).toHaveBeenCalledTimes(1); - const mockInstance = vi.mocked(OpenAIPlugin).mock.results[0].value; - expect(mockInstance.enable).toHaveBeenCalledTimes(1); - }); - - it("should be idempotent on multiple disable calls", () => { - const plugin = new BraintrustPlugin(); - plugin.enable(); - - const openaiMock = vi.mocked(OpenAIPlugin).mock.results[0].value; - - plugin.disable(); - plugin.disable(); - plugin.disable(); - - // Should only disable plugins once - expect(openaiMock.disable).toHaveBeenCalledTimes(1); - }); - - it("should not disable plugins if never enabled", () => { - const plugin = new BraintrustPlugin(); - plugin.disable(); - - // Should not create any plugins - expect(OpenAIPlugin).not.toHaveBeenCalled(); - expect(OpenAICodexPlugin).not.toHaveBeenCalled(); - expect(AnthropicPlugin).not.toHaveBeenCalled(); - expect(AISDKPlugin).not.toHaveBeenCalled(); - expect(ClaudeAgentSDKPlugin).not.toHaveBeenCalled(); - expect(OpenAIAgentsPlugin).not.toHaveBeenCalled(); - expect(GoogleGenAIPlugin).not.toHaveBeenCalled(); - expect(HuggingFacePlugin).not.toHaveBeenCalled(); - expect(OpenRouterPlugin).not.toHaveBeenCalled(); - expect(OpenRouterAgentPlugin).not.toHaveBeenCalled(); - expect(MistralPlugin).not.toHaveBeenCalled(); - expect(OllamaPlugin).not.toHaveBeenCalled(); - expect(CoherePlugin).not.toHaveBeenCalled(); - expect(GroqPlugin).not.toHaveBeenCalled(); - expect(PiCodingAgentPlugin).not.toHaveBeenCalled(); - expect(StrandsAgentSDKPlugin).not.toHaveBeenCalled(); - }); - - it("should allow re-enabling after disable", () => { - const plugin = new BraintrustPlugin(); - plugin.enable(); - plugin.disable(); - - vi.clearAllMocks(); - - plugin.enable(); - - expect(OpenAIPlugin).toHaveBeenCalledTimes(1); - expect(OpenAICodexPlugin).toHaveBeenCalledTimes(1); - expect(AnthropicPlugin).toHaveBeenCalledTimes(1); - expect(AISDKPlugin).toHaveBeenCalledTimes(1); - expect(ClaudeAgentSDKPlugin).toHaveBeenCalledTimes(1); - expect(OpenAIAgentsPlugin).toHaveBeenCalledTimes(1); - expect(GoogleGenAIPlugin).toHaveBeenCalledTimes(1); - expect(HuggingFacePlugin).toHaveBeenCalledTimes(1); - expect(OpenRouterPlugin).toHaveBeenCalledTimes(1); - expect(OpenRouterAgentPlugin).toHaveBeenCalledTimes(1); - expect(MistralPlugin).toHaveBeenCalledTimes(1); - expect(OllamaPlugin).toHaveBeenCalledTimes(1); - expect(CoherePlugin).toHaveBeenCalledTimes(1); - expect(GroqPlugin).toHaveBeenCalledTimes(1); - expect(PiCodingAgentPlugin).toHaveBeenCalledTimes(1); - expect(StrandsAgentSDKPlugin).toHaveBeenCalledTimes(1); - expect(LangChainPlugin).toHaveBeenCalledTimes(1); - expect(LangSmithPlugin).toHaveBeenCalledTimes(1); - }); - - it("should only disable plugins that were enabled", () => { - const plugin = new BraintrustPlugin({ - integrations: { - openai: true, - anthropic: false, - aisdk: true, - claudeAgentSDK: false, - openAIAgents: true, - googleGenAI: true, - huggingface: true, - openrouter: true, - openrouterAgent: true, - mistral: false, - ollama: false, - cohere: false, - groq: true, - langchain: true, - }, - }); - plugin.enable(); - - const openaiMock = vi.mocked(OpenAIPlugin).mock.results[0].value; - const aiSDKMock = vi.mocked(AISDKPlugin).mock.results[0].value; - const openAIAgentsMock = - vi.mocked(OpenAIAgentsPlugin).mock.results[0].value; - const googleGenAIMock = - vi.mocked(GoogleGenAIPlugin).mock.results[0].value; - const huggingFaceMock = - vi.mocked(HuggingFacePlugin).mock.results[0].value; - const openRouterMock = vi.mocked(OpenRouterPlugin).mock.results[0].value; - const openRouterAgentMock = vi.mocked(OpenRouterAgentPlugin).mock - .results[0].value; - const groqMock = vi.mocked(GroqPlugin).mock.results[0].value; - const langChainMock = vi.mocked(LangChainPlugin).mock.results[0].value; - - plugin.disable(); - - expect(openaiMock.disable).toHaveBeenCalledTimes(1); - expect(aiSDKMock.disable).toHaveBeenCalledTimes(1); - expect(openAIAgentsMock.disable).toHaveBeenCalledTimes(1); - expect(googleGenAIMock.disable).toHaveBeenCalledTimes(1); - expect(huggingFaceMock.disable).toHaveBeenCalledTimes(1); - expect(openRouterMock.disable).toHaveBeenCalledTimes(1); - expect(openRouterAgentMock.disable).toHaveBeenCalledTimes(1); - expect(groqMock.disable).toHaveBeenCalledTimes(1); - expect(langChainMock.disable).toHaveBeenCalledTimes(1); - expect(MistralPlugin).not.toHaveBeenCalled(); - expect(OllamaPlugin).not.toHaveBeenCalled(); - expect(CoherePlugin).not.toHaveBeenCalled(); - }); - - it("should not create Voyage AI plugin when voyageai: false", () => { - const plugin = new BraintrustPlugin({ - integrations: { voyageai: false }, - }); - plugin.enable(); - - expect(VoyageAIPlugin).not.toHaveBeenCalled(); - }); - }); -}); - -// Re-exported utility function tests from OpenAI plugin -import { - parseMetricsFromUsage, - processImagesInOutput, - aggregateChatCompletionChunks, -} from "./braintrust-plugin"; -import { Attachment } from "../logger"; - -describe("parseMetricsFromUsage", () => { - it("should return empty object for null usage", () => { - expect(parseMetricsFromUsage(null)).toEqual({}); - }); - - it("should return empty object for undefined usage", () => { - expect(parseMetricsFromUsage(undefined)).toEqual({}); - }); - - it("should parse legacy token names", () => { - const usage = { - prompt_tokens: 10, - completion_tokens: 20, - total_tokens: 30, - }; - - const result = parseMetricsFromUsage(usage); - - expect(result).toEqual({ - prompt_tokens: 10, - completion_tokens: 20, - tokens: 30, // total_tokens is mapped to tokens - }); - }); - - it("should parse new API token names and map them", () => { - const usage = { - input_tokens: 15, - output_tokens: 25, - total_tokens: 40, - }; - - const result = parseMetricsFromUsage(usage); - - expect(result).toEqual({ - prompt_tokens: 15, // input_tokens mapped to prompt_tokens - completion_tokens: 25, // output_tokens mapped to completion_tokens - tokens: 40, // total_tokens mapped to tokens - }); - }); - - it("should parse token details fields", () => { - const usage = { - input_tokens: 100, - input_tokens_details: { - cached_tokens: 50, - audio_tokens: 10, - }, - output_tokens: 80, - output_tokens_details: { - reasoning_tokens: 20, - }, - }; - - const result = parseMetricsFromUsage(usage); - - expect(result).toEqual({ - prompt_tokens: 100, - prompt_cached_tokens: 50, // input -> prompt prefix - prompt_audio_tokens: 10, - completion_tokens: 80, - completion_reasoning_tokens: 20, // output -> completion prefix - }); - }); - - it("should handle mixed legacy and new token names", () => { - const usage = { - prompt_tokens: 10, - input_tokens: 15, - completion_tokens: 20, - total_tokens: 30, - }; - - const result = parseMetricsFromUsage(usage); - - expect(result).toEqual({ - prompt_tokens: 15, // Last one wins (input_tokens mapped) - completion_tokens: 20, - tokens: 30, // total_tokens is mapped to tokens - }); - }); - - it("should ignore non-number token values", () => { - const usage = { - prompt_tokens: "not a number", - completion_tokens: 20, - total_tokens: null, - }; - - const result = parseMetricsFromUsage(usage); - - expect(result).toEqual({ - completion_tokens: 20, - }); - }); - - it("should ignore non-object token details", () => { - const usage = { - input_tokens: 100, - input_tokens_details: "not an object", - output_tokens: 80, - }; - - const result = parseMetricsFromUsage(usage); - - expect(result).toEqual({ - prompt_tokens: 100, - completion_tokens: 80, - }); - }); - - it("should ignore non-number values in token details", () => { - const usage = { - input_tokens: 100, - input_tokens_details: { - cached_tokens: 50, - invalid: "not a number", - also_invalid: null, - }, - }; - - const result = parseMetricsFromUsage(usage); - - expect(result).toEqual({ - prompt_tokens: 100, - prompt_cached_tokens: 50, - }); - }); - - it("should handle empty usage object", () => { - expect(parseMetricsFromUsage({})).toEqual({}); - }); - - it("should handle unknown token prefix in details", () => { - const usage = { - custom_tokens: 100, - custom_tokens_details: { - special: 25, - }, - }; - - const result = parseMetricsFromUsage(usage); - - expect(result).toEqual({ - custom_tokens: 100, - custom_special: 25, // custom prefix preserved - }); - }); -}); - -describe("processImagesInOutput", () => { - it("should convert image_generation_call to attachment", () => { - // Create a small 1x1 red PNG base64 - const base64Image = - "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8DwHwAFBQIAX8jx0gAAAABJRU5ErkJggg=="; - - const output = { - type: "image_generation_call", - result: base64Image, - output_format: "png", - revised_prompt: "A red pixel", - }; - - const result = processImagesInOutput(output); - - expect(result.type).toBe("image_generation_call"); - expect(result.output_format).toBe("png"); - expect(result.revised_prompt).toBe("A red pixel"); - expect(result.result).toBeInstanceOf(Attachment); - - const attachment = result.result as Attachment; - expect(attachment.reference.filename).toContain(".png"); - expect(attachment.reference.content_type).toBe("image/png"); - }); - - it("should handle image_generation_call with long revised_prompt", () => { - const longPrompt = - "This is a very long prompt that should be truncated to 50 characters when used as filename"; - const base64Image = - "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8DwHwAFBQIAX8jx0gAAAABJRU5ErkJggg=="; - - const output = { - type: "image_generation_call", - result: base64Image, - revised_prompt: longPrompt, - }; - - const result = processImagesInOutput(output); - - const attachment = result.result as Attachment; - // Should be truncated to 50 chars + sanitized + extension - expect(attachment.reference.filename.length).toBeLessThanOrEqual(54); // 50 + ".png" - }); - - it("should handle image_generation_call without revised_prompt", () => { - const base64Image = - "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8DwHwAFBQIAX8jx0gAAAABJRU5ErkJggg=="; - - const output = { - type: "image_generation_call", - result: base64Image, - output_format: "jpg", - }; - - const result = processImagesInOutput(output); - - const attachment = result.result as Attachment; - expect(attachment.reference.filename).toBe("generated_image.jpg"); - expect(attachment.reference.content_type).toBe("image/jpg"); - }); - - it("should use png as default format", () => { - const base64Image = - "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8DwHwAFBQIAX8jx0gAAAABJRU5ErkJggg=="; - - const output = { - type: "image_generation_call", - result: base64Image, - }; - - const result = processImagesInOutput(output); - - const attachment = result.result as Attachment; - expect(attachment.reference.filename).toBe("generated_image.png"); - expect(attachment.reference.content_type).toBe("image/png"); - }); - - it("should process arrays recursively", () => { - const base64Image = - "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8DwHwAFBQIAX8jx0gAAAABJRU5ErkJggg=="; - - const output = [ - { type: "text", content: "Hello" }, - { - type: "image_generation_call", - result: base64Image, - }, - { type: "text", content: "World" }, - ]; - - const result = processImagesInOutput(output); - - expect(Array.isArray(result)).toBe(true); - expect(result[0]).toEqual({ type: "text", content: "Hello" }); - expect(result[1].result).toBeInstanceOf(Attachment); - expect(result[2]).toEqual({ type: "text", content: "World" }); - }); - - it("should pass through non-image objects unchanged", () => { - const output = { - type: "text", - content: "Hello world", - }; - - const result = processImagesInOutput(output); - expect(result).toEqual(output); - }); - - it("should pass through primitive values unchanged", () => { - expect(processImagesInOutput("string")).toBe("string"); - expect(processImagesInOutput(42)).toBe(42); - expect(processImagesInOutput(true)).toBe(true); - expect(processImagesInOutput(null)).toBe(null); - }); - - it("should not process image_generation_call with non-string result", () => { - const output = { - type: "image_generation_call", - result: 12345, // Not a string - }; - - const result = processImagesInOutput(output); - expect(result).toEqual(output); - expect(result.result).toBe(12345); - }); -}); - -describe("aggregateChatCompletionChunks", () => { - it("should aggregate simple text chunks", () => { - const chunks = [ - { - choices: [{ delta: { role: "assistant", content: "Hello" } }], - }, - { - choices: [{ delta: { content: " world" } }], - }, - { - choices: [{ delta: { content: "!" } }], - }, - ]; - - const result = aggregateChatCompletionChunks(chunks as any); - - expect(result.output).toEqual([ - { - index: 0, - message: { - role: "assistant", - content: "Hello world!", - tool_calls: undefined, - }, - logprobs: null, - finish_reason: undefined, - }, - ]); - expect(result.metrics).toEqual({}); - }); - - it("should extract role from first chunk", () => { - const chunks = [ - { - choices: [{ delta: { role: "assistant" } }], - }, - { - choices: [{ delta: { content: "Hi" } }], - }, - ]; - - const result = aggregateChatCompletionChunks(chunks as any); - - expect(result.output[0].message.role).toBe("assistant"); - }); - - it("should extract finish_reason from last chunk with it", () => { - const chunks = [ - { - choices: [{ delta: { role: "assistant", content: "Done" } }], - }, - { - choices: [{ delta: { finish_reason: "stop" } }], - }, - ]; - - const result = aggregateChatCompletionChunks(chunks); - - expect(result.output[0].finish_reason).toBe("stop"); - }); - - it("should aggregate tool calls by id", () => { - const chunks = [ - { - choices: [ - { - delta: { - role: "assistant", - tool_calls: [ - { - id: "call_1", - type: "function", - function: { name: "get_weather", arguments: '{"loc' }, - }, - ], - }, - }, - ], - }, - { - choices: [ - { - delta: { - tool_calls: [ - { - function: { arguments: 'ation":"' }, - }, - ], - }, - }, - ], - }, - { - choices: [ - { - delta: { - tool_calls: [ - { - function: { arguments: 'NYC"}' }, - }, - ], - }, - }, - ], - }, - ]; - - const result = aggregateChatCompletionChunks(chunks); - - expect(result.output[0].message.tool_calls).toEqual([ - { - id: "call_1", - type: "function", - function: { name: "get_weather", arguments: '{"location":"NYC"}' }, - }, - ]); - }); - - it("should handle multiple tool calls", () => { - const chunks = [ - { - choices: [ - { - delta: { - role: "assistant", - tool_calls: [ - { - id: "call_1", - type: "function", - function: { name: "tool1", arguments: '{"a":' }, - }, - ], - }, - }, - ], - }, - { - choices: [ - { - delta: { - tool_calls: [ - { - function: { arguments: "1}" }, - }, - ], - }, - }, - ], - }, - { - choices: [ - { - delta: { - tool_calls: [ - { - id: "call_2", - type: "function", - function: { name: "tool2", arguments: '{"b":' }, - }, - ], - }, - }, - ], - }, - { - choices: [ - { - delta: { - tool_calls: [ - { - function: { arguments: "2}" }, - }, - ], - }, - }, - ], - }, - ]; - - const result = aggregateChatCompletionChunks(chunks as any); - const output = result.output as any[]; - - expect(output[0].message.tool_calls).toHaveLength(2); - expect(output[0].message.tool_calls[0]).toEqual({ - id: "call_1", - type: "function", - function: { name: "tool1", arguments: '{"a":1}' }, - }); - expect(output[0].message.tool_calls[1]).toEqual({ - id: "call_2", - type: "function", - function: { name: "tool2", arguments: '{"b":2}' }, - }); - }); - - it("should parse usage metrics from chunks", () => { - const chunks = [ - { - choices: [{ delta: { role: "assistant", content: "Hi" } }], - }, - { - choices: [{ delta: { content: "!" } }], - usage: { - prompt_tokens: 10, - completion_tokens: 2, - total_tokens: 12, - }, - }, - ]; - - const result = aggregateChatCompletionChunks(chunks as any); - - expect(result.metrics).toEqual({ - prompt_tokens: 10, - completion_tokens: 2, - tokens: 12, // total_tokens is mapped to tokens - }); - }); - - it("should merge usage from multiple chunks", () => { - const chunks = [ - { - choices: [{ delta: { role: "assistant" } }], - usage: { - prompt_tokens: 10, - }, - }, - { - choices: [{ delta: { content: "Hi" } }], - usage: { - completion_tokens: 5, - total_tokens: 15, - }, - }, - ]; - - const result = aggregateChatCompletionChunks(chunks); - - expect(result.metrics).toEqual({ - prompt_tokens: 10, - completion_tokens: 5, - tokens: 15, // total_tokens is mapped to tokens - }); - }); - - it("should handle empty chunks array", () => { - const result = aggregateChatCompletionChunks([]); - - expect(result.output).toEqual([ - { - index: 0, - message: { - role: undefined, - content: undefined, - tool_calls: undefined, - }, - logprobs: null, - finish_reason: undefined, - }, - ]); - expect(result.metrics).toEqual({}); - }); - - it("should handle chunks without choices", () => { - const chunks = [ - {}, - { choices: null }, - { choices: [] }, - { choices: [{ delta: { content: "Hi" } }] }, - ]; - - const result = aggregateChatCompletionChunks(chunks as any); - - expect(result.output[0].message.content).toBe("Hi"); - }); - - it("should handle chunks with only usage", () => { - const chunks = [ - { - usage: { - prompt_tokens: 10, - completion_tokens: 5, - }, - }, - ]; - - const result = aggregateChatCompletionChunks(chunks); - - expect(result.metrics).toEqual({ - prompt_tokens: 10, - completion_tokens: 5, - }); - }); - - it("should handle mixed content and tool calls", () => { - const chunks = [ - { - choices: [ - { - delta: { - role: "assistant", - content: "Let me check", - }, - }, - ], - }, - { - choices: [ - { - delta: { - tool_calls: [ - { - id: "call_1", - type: "function", - function: { name: "check", arguments: "{}" }, - }, - ], - }, - }, - ], - }, - { - choices: [{ delta: { finish_reason: "tool_calls" } }], - }, - ]; - - const result = aggregateChatCompletionChunks(chunks); - - expect(result.output[0].message.content).toBe("Let me check"); - expect(result.output[0].message.tool_calls).toHaveLength(1); - expect(result.output[0].finish_reason).toBe("tool_calls"); - }); -}); diff --git a/js/src/instrumentation/braintrust-plugin.ts b/js/src/instrumentation/braintrust-plugin.ts deleted file mode 100644 index 53bebc00d..000000000 --- a/js/src/instrumentation/braintrust-plugin.ts +++ /dev/null @@ -1,402 +0,0 @@ -import { BasePlugin } from "./core"; -import { OpenAIPlugin } from "./plugins/openai-plugin"; -import { OpenAICodexPlugin } from "./plugins/openai-codex-plugin"; -import { AnthropicPlugin } from "./plugins/anthropic-plugin"; -import { AISDKPlugin } from "./plugins/ai-sdk-plugin"; -import { ClaudeAgentSDKPlugin } from "./plugins/claude-agent-sdk-plugin"; -import { CloudflareThinkPlugin } from "./plugins/cloudflare-think-plugin"; -import { CursorSDKPlugin } from "./plugins/cursor-sdk-plugin"; -import { OpenAIAgentsPlugin } from "./plugins/openai-agents-plugin"; -import { GoogleGenAIPlugin } from "./plugins/google-genai-plugin"; -import { HuggingFacePlugin } from "./plugins/huggingface-plugin"; -import { HuggingFaceTransformersPlugin } from "./plugins/huggingface-transformers-plugin"; -import { OpenRouterAgentPlugin } from "./plugins/openrouter-agent-plugin"; -import { OpenRouterPlugin } from "./plugins/openrouter-plugin"; -import { MistralPlugin } from "./plugins/mistral-plugin"; -import { OllamaPlugin } from "./plugins/ollama-plugin"; -import { GoogleADKPlugin } from "./plugins/google-adk-plugin"; -import { CoherePlugin } from "./plugins/cohere-plugin"; -import { GroqPlugin } from "./plugins/groq-plugin"; -import { BedrockRuntimePlugin } from "./plugins/bedrock-runtime-plugin"; -import { GenkitPlugin } from "./plugins/genkit-plugin"; -import { GitHubCopilotPlugin } from "./plugins/github-copilot-plugin"; -import { LangChainPlugin } from "./plugins/langchain-plugin"; -import { LangSmithPlugin } from "./plugins/langsmith-plugin"; -import { PiCodingAgentPlugin } from "./plugins/pi-coding-agent-plugin"; -import { StrandsAgentSDKPlugin } from "./plugins/strands-agent-sdk-plugin"; -import { VoyageAIPlugin } from "./plugins/voyageai-plugin"; -import { CloudflareAIChatPlugin } from "./plugins/cloudflare-ai-chat-plugin"; -import { CloudflareAgentsPlugin } from "./plugins/cloudflare-agents-plugin"; -import type { InstrumentationIntegrationsConfig } from "./config"; - -export interface BraintrustPluginConfig { - integrations?: InstrumentationIntegrationsConfig; -} - -/** - * Default Braintrust plugin that manages all AI provider instrumentation plugins. - * - * This plugin orchestrates: - * - OpenAI SDK (chat completions, embeddings, etc.) - * - Anthropic SDK (messages) - * - Claude Agent SDK (agent interactions) - * - Vercel AI SDK (generateText, streamText, etc.) - * - Google GenAI SDK - * - HuggingFace Inference SDK - * - LangChain.js and LangGraph - * - Mistral SDK - * - Ollama SDK - * - Cohere SDK - * - Voyage AI SDK - * - * The plugin is automatically enabled when the Braintrust library is loaded. - * Individual integrations can be disabled via configuration. - */ -export class BraintrustPlugin extends BasePlugin { - private config: BraintrustPluginConfig; - private openaiPlugin: OpenAIPlugin | null = null; - private openAICodexPlugin: OpenAICodexPlugin | null = null; - private anthropicPlugin: AnthropicPlugin | null = null; - private aiSDKPlugin: AISDKPlugin | null = null; - private claudeAgentSDKPlugin: ClaudeAgentSDKPlugin | null = null; - private cloudflareThinkPlugin: CloudflareThinkPlugin | null = null; - private cursorSDKPlugin: CursorSDKPlugin | null = null; - private openAIAgentsPlugin: OpenAIAgentsPlugin | null = null; - private googleGenAIPlugin: GoogleGenAIPlugin | null = null; - private huggingFacePlugin: HuggingFacePlugin | null = null; - private huggingFaceTransformersPlugin: HuggingFaceTransformersPlugin | null = - null; - private openRouterPlugin: OpenRouterPlugin | null = null; - private openRouterAgentPlugin: OpenRouterAgentPlugin | null = null; - private mistralPlugin: MistralPlugin | null = null; - private ollamaPlugin: OllamaPlugin | null = null; - private googleADKPlugin: GoogleADKPlugin | null = null; - private coherePlugin: CoherePlugin | null = null; - private groqPlugin: GroqPlugin | null = null; - private bedrockRuntimePlugin: BedrockRuntimePlugin | null = null; - private genkitPlugin: GenkitPlugin | null = null; - private gitHubCopilotPlugin: GitHubCopilotPlugin | null = null; - private langChainPlugin: LangChainPlugin | null = null; - private langSmithPlugin: LangSmithPlugin | null = null; - private piCodingAgentPlugin: PiCodingAgentPlugin | null = null; - private strandsAgentSDKPlugin: StrandsAgentSDKPlugin | null = null; - private voyageAIPlugin: VoyageAIPlugin | null = null; - private cloudflareAIChatPlugin: CloudflareAIChatPlugin | null = null; - private cloudflareAgentsPlugin: CloudflareAgentsPlugin | null = null; - - constructor(config: BraintrustPluginConfig = {}) { - super(); - this.config = config; - } - - protected onEnable(): void { - const integrations = this.config.integrations ?? {}; - - // Enable OpenAI integration (default: true) - if (integrations.openai !== false) { - this.openaiPlugin = new OpenAIPlugin(); - this.openaiPlugin.enable(); - } - - if (integrations.openaiCodexSDK !== false) { - this.openAICodexPlugin = new OpenAICodexPlugin(); - this.openAICodexPlugin.enable(); - } - - // Enable Anthropic integration (default: true) - if (integrations.anthropic !== false) { - this.anthropicPlugin = new AnthropicPlugin(); - this.anthropicPlugin.enable(); - } - - // Enable AI SDK integration (default: true) - // Support both 'aisdk' and legacy 'vercel' config keys - if (integrations.aisdk !== false && integrations.vercel !== false) { - this.aiSDKPlugin = new AISDKPlugin(); - this.aiSDKPlugin.enable(); - } - - // Enable Claude Agent SDK integration (default: true) - if (integrations.claudeAgentSDK !== false) { - this.claudeAgentSDKPlugin = new ClaudeAgentSDKPlugin(); - this.claudeAgentSDKPlugin.enable(); - } - - if (integrations.cloudflareThink !== false) { - this.cloudflareThinkPlugin = new CloudflareThinkPlugin(); - this.cloudflareThinkPlugin.enable(); - } - - if (integrations.cursorSDK !== false && integrations.cursor !== false) { - this.cursorSDKPlugin = new CursorSDKPlugin(); - this.cursorSDKPlugin.enable(); - } - - // Enable OpenAI Agents SDK integration (default: true) - if (integrations.openAIAgents !== false) { - this.openAIAgentsPlugin = new OpenAIAgentsPlugin(); - this.openAIAgentsPlugin.enable(); - } - - // Enable Google GenAI integration (default: true) - // Support both 'googleGenAI' and legacy 'google' config keys - if (integrations.googleGenAI !== false && integrations.google !== false) { - this.googleGenAIPlugin = new GoogleGenAIPlugin(); - this.googleGenAIPlugin.enable(); - } - - if (integrations.huggingface !== false) { - this.huggingFacePlugin = new HuggingFacePlugin(); - this.huggingFacePlugin.enable(); - this.huggingFaceTransformersPlugin = new HuggingFaceTransformersPlugin(); - this.huggingFaceTransformersPlugin.enable(); - } - - if (integrations.openrouter !== false) { - this.openRouterPlugin = new OpenRouterPlugin(); - this.openRouterPlugin.enable(); - } - - if (integrations.openrouterAgent !== false) { - this.openRouterAgentPlugin = new OpenRouterAgentPlugin(); - this.openRouterAgentPlugin.enable(); - } - - if (integrations.mistral !== false) { - this.mistralPlugin = new MistralPlugin(); - this.mistralPlugin.enable(); - } - - if (integrations.ollama !== false) { - this.ollamaPlugin = new OllamaPlugin(); - this.ollamaPlugin.enable(); - } - - // Enable Google ADK integration (default: true) - if (integrations.googleADK !== false) { - this.googleADKPlugin = new GoogleADKPlugin(); - this.googleADKPlugin.enable(); - } - - if (integrations.cohere !== false) { - this.coherePlugin = new CoherePlugin(); - this.coherePlugin.enable(); - } - - if (integrations.voyageai !== false) { - this.voyageAIPlugin = new VoyageAIPlugin(); - this.voyageAIPlugin.enable(); - } - - if (integrations.groq !== false) { - this.groqPlugin = new GroqPlugin(); - this.groqPlugin.enable(); - } - - if ( - integrations.bedrock !== false && - integrations.awsBedrock !== false && - integrations.awsBedrockRuntime !== false - ) { - this.bedrockRuntimePlugin = new BedrockRuntimePlugin(); - this.bedrockRuntimePlugin.enable(); - } - - if (integrations.genkit !== false) { - this.genkitPlugin = new GenkitPlugin(); - this.genkitPlugin.enable(); - } - - if (integrations.gitHubCopilot !== false) { - this.gitHubCopilotPlugin = new GitHubCopilotPlugin(); - this.gitHubCopilotPlugin.enable(); - } - - if (integrations.piCodingAgent !== false) { - this.piCodingAgentPlugin = new PiCodingAgentPlugin(); - this.piCodingAgentPlugin.enable(); - } - - if (integrations.strandsAgentSDK !== false) { - this.strandsAgentSDKPlugin = new StrandsAgentSDKPlugin(); - this.strandsAgentSDKPlugin.enable(); - } - - if (integrations.cloudflareAIChat !== false) { - this.cloudflareAIChatPlugin = new CloudflareAIChatPlugin(); - this.cloudflareAIChatPlugin.enable(); - } - - if (integrations.cloudflareAgents !== false) { - this.cloudflareAgentsPlugin = new CloudflareAgentsPlugin(); - this.cloudflareAgentsPlugin.enable(); - } - - if (integrations.langchain !== false && integrations.langgraph !== false) { - this.langChainPlugin = new LangChainPlugin(); - this.langChainPlugin.enable(); - } - - if (integrations.langsmith !== false) { - this.langSmithPlugin = new LangSmithPlugin({ - skipLangChainRuns: integrations.langchain !== false, - }); - this.langSmithPlugin.enable(); - } - - // Mastra is intentionally not wired here: `@mastra/core` ships its own - // ObservabilityExporter contract, and `BraintrustObservabilityExporter` - // (wrappers/mastra.ts) is auto-installed by the loader patch in - // `auto-instrumentations/loader/mastra-observability-patch.ts` rather than - // by a BasePlugin / global hook subscription. - } - - protected onDisable(): void { - if (this.openaiPlugin) { - this.openaiPlugin.disable(); - this.openaiPlugin = null; - } - - if (this.openAICodexPlugin) { - this.openAICodexPlugin.disable(); - this.openAICodexPlugin = null; - } - - if (this.anthropicPlugin) { - this.anthropicPlugin.disable(); - this.anthropicPlugin = null; - } - - if (this.aiSDKPlugin) { - this.aiSDKPlugin.disable(); - this.aiSDKPlugin = null; - } - - if (this.claudeAgentSDKPlugin) { - this.claudeAgentSDKPlugin.disable(); - this.claudeAgentSDKPlugin = null; - } - - if (this.cloudflareThinkPlugin) { - this.cloudflareThinkPlugin.disable(); - this.cloudflareThinkPlugin = null; - } - - if (this.cursorSDKPlugin) { - this.cursorSDKPlugin.disable(); - this.cursorSDKPlugin = null; - } - - if (this.openAIAgentsPlugin) { - this.openAIAgentsPlugin.disable(); - this.openAIAgentsPlugin = null; - } - - if (this.googleGenAIPlugin) { - this.googleGenAIPlugin.disable(); - this.googleGenAIPlugin = null; - } - - if (this.huggingFacePlugin) { - this.huggingFacePlugin.disable(); - this.huggingFacePlugin = null; - } - - if (this.huggingFaceTransformersPlugin) { - this.huggingFaceTransformersPlugin.disable(); - this.huggingFaceTransformersPlugin = null; - } - - if (this.openRouterPlugin) { - this.openRouterPlugin.disable(); - this.openRouterPlugin = null; - } - - if (this.openRouterAgentPlugin) { - this.openRouterAgentPlugin.disable(); - this.openRouterAgentPlugin = null; - } - - if (this.mistralPlugin) { - this.mistralPlugin.disable(); - this.mistralPlugin = null; - } - - if (this.ollamaPlugin) { - this.ollamaPlugin.disable(); - this.ollamaPlugin = null; - } - - if (this.googleADKPlugin) { - this.googleADKPlugin.disable(); - this.googleADKPlugin = null; - } - - if (this.coherePlugin) { - this.coherePlugin.disable(); - this.coherePlugin = null; - } - - if (this.voyageAIPlugin) { - this.voyageAIPlugin.disable(); - this.voyageAIPlugin = null; - } - - if (this.groqPlugin) { - this.groqPlugin.disable(); - this.groqPlugin = null; - } - - if (this.bedrockRuntimePlugin) { - this.bedrockRuntimePlugin.disable(); - this.bedrockRuntimePlugin = null; - } - - if (this.genkitPlugin) { - this.genkitPlugin.disable(); - this.genkitPlugin = null; - } - - if (this.gitHubCopilotPlugin) { - this.gitHubCopilotPlugin.disable(); - this.gitHubCopilotPlugin = null; - } - - if (this.piCodingAgentPlugin) { - this.piCodingAgentPlugin.disable(); - this.piCodingAgentPlugin = null; - } - - if (this.strandsAgentSDKPlugin) { - this.strandsAgentSDKPlugin.disable(); - this.strandsAgentSDKPlugin = null; - } - - if (this.cloudflareAIChatPlugin) { - this.cloudflareAIChatPlugin.disable(); - this.cloudflareAIChatPlugin = null; - } - - if (this.cloudflareAgentsPlugin) { - this.cloudflareAgentsPlugin.disable(); - this.cloudflareAgentsPlugin = null; - } - - if (this.langChainPlugin) { - this.langChainPlugin.disable(); - this.langChainPlugin = null; - } - - if (this.langSmithPlugin) { - this.langSmithPlugin.disable(); - this.langSmithPlugin = null; - } - } -} - -// Re-export utility functions from OpenAIPlugin for backward compatibility -export { - parseMetricsFromUsage, - processImagesInOutput, - aggregateChatCompletionChunks, -} from "./plugins/openai-plugin"; diff --git a/js/src/instrumentation/core/channel-tracing.test.ts b/js/src/instrumentation/core/channel-tracing.test.ts index 55c08f05b..319d24344 100644 --- a/js/src/instrumentation/core/channel-tracing.test.ts +++ b/js/src/instrumentation/core/channel-tracing.test.ts @@ -27,12 +27,32 @@ import { traceAsyncChannel, traceStreamingChannel } from "./channel-tracing"; const testChannels = defineChannels( "channel-tracing-test", { - asyncCall: channel<[Record], { ok: true }>({ - channelName: "async.call", + asyncBinding: channel<[Record], { ok: true }>({ + channelName: "async.binding", kind: "async", }), - streamingCall: channel<[Record], { ok: true }>({ - channelName: "streaming.call", + provenance: channel<[Record], { ok: true }>({ + channelName: "async.provenance", + kind: "async", + }), + skipped: channel<[Record], { ok: true }>({ + channelName: "async.skipped", + kind: "async", + }), + throwingPredicate: channel<[Record], { ok: true }>({ + channelName: "async.throwing-predicate", + kind: "async", + }), + suppressed: channel<[Record], { ok: true }>({ + channelName: "async.suppressed", + kind: "async", + }), + streamingCleanup: channel<[Record], { ok: true }>({ + channelName: "streaming.cleanup", + kind: "async", + }), + streamingCancellation: channel<[Record], { ok: true }>({ + channelName: "streaming.cancellation", kind: "async", }), }, @@ -60,7 +80,7 @@ describe("traceAsyncChannel current span binding", () => { }); it("binds the created span into the traced async execution context", async () => { - const unsubscribe = traceAsyncChannel(testChannels.asyncCall, { + traceAsyncChannel(testChannels.asyncBinding, { name: "channel-tracing-test", type: "function", extractInput: () => ({ @@ -73,21 +93,16 @@ describe("traceAsyncChannel current span binding", () => { const seenSpanIds: string[] = []; - try { - await testChannels.asyncCall.tracePromise( - async () => { - seenSpanIds.push(currentSpan().spanId); - await Promise.resolve(); - seenSpanIds.push(currentSpan().spanId); - - return { ok: true as const }; - }, - { arguments: [{}] } as any, - ); - } finally { - unsubscribe(); - } + await testChannels.asyncBinding.tracePromise( + async () => { + seenSpanIds.push(currentSpan().spanId); + await Promise.resolve(); + seenSpanIds.push(currentSpan().spanId); + return { ok: true as const }; + }, + { arguments: [{}] } as any, + ); expect(seenSpanIds).toHaveLength(2); expect(seenSpanIds[0]).toBeTruthy(); expect(seenSpanIds[1]).toBe(seenSpanIds[0]); @@ -103,7 +118,7 @@ describe("traceAsyncChannel current span binding", () => { }); it("limits channel provenance to directly instrumented spans", async () => { - const unsubscribe = traceAsyncChannel(testChannels.asyncCall, { + traceAsyncChannel(testChannels.provenance, { name: "channel-parent", type: "function", extractInput: () => ({ input: "input", metadata: undefined }), @@ -111,32 +126,27 @@ describe("traceAsyncChannel current span binding", () => { extractMetrics: () => ({}), }); - try { - await testChannels.asyncCall.tracePromise( - async () => { - const parent = currentSpan(); - parent.startSpan({ name: "user-child" }).end(); - parent - .startSpanWithParents("user-multi-parent-child", [parent.spanId], { - name: "user-multi-parent-child", - }) - .end(); - parent - .startSpan( - withSpanInstrumentationName( - { name: "instrumentation-child" }, - INSTRUMENTATION_NAMES.OPENAI, - ), - ) - .end(); - return { ok: true as const }; - }, - { arguments: [{}] } as any, - ); - } finally { - unsubscribe(); - } - + await testChannels.provenance.tracePromise( + async () => { + const parent = currentSpan(); + parent.startSpan({ name: "user-child" }).end(); + parent + .startSpanWithParents("user-multi-parent-child", [parent.spanId], { + name: "user-multi-parent-child", + }) + .end(); + parent + .startSpan( + withSpanInstrumentationName( + { name: "instrumentation-child" }, + INSTRUMENTATION_NAMES.OPENAI, + ), + ) + .end(); + return { ok: true as const }; + }, + { arguments: [{}] } as any, + ); const spans = (await backgroundLogger.drain()) as Array< Record >; @@ -161,7 +171,7 @@ describe("traceAsyncChannel current span binding", () => { }); it("does not create a span when shouldTrace returns false", async () => { - const unsubscribe = traceAsyncChannel(testChannels.asyncCall, { + traceAsyncChannel(testChannels.skipped, { name: "channel-tracing-test", shouldTrace: ([params]) => !( @@ -181,21 +191,16 @@ describe("traceAsyncChannel current span binding", () => { const seenSpanIds: string[] = []; - try { - await testChannels.asyncCall.tracePromise( - async () => { - seenSpanIds.push(currentSpan().spanId); - await Promise.resolve(); - seenSpanIds.push(currentSpan().spanId); - - return { ok: true as const }; - }, - { arguments: [{ skip: true }] } as any, - ); - } finally { - unsubscribe(); - } + await testChannels.skipped.tracePromise( + async () => { + seenSpanIds.push(currentSpan().spanId); + await Promise.resolve(); + seenSpanIds.push(currentSpan().spanId); + return { ok: true as const }; + }, + { arguments: [{ skip: true }] } as any, + ); expect(seenSpanIds).toEqual(["", ""]); expect(currentSpan()).toBe(NOOP_SPAN); @@ -207,7 +212,7 @@ describe("traceAsyncChannel current span binding", () => { const consoleErrorSpy = vi .spyOn(console, "error") .mockImplementation(() => {}); - const unsubscribe = traceAsyncChannel(testChannels.asyncCall, { + traceAsyncChannel(testChannels.throwingPredicate, { name: "channel-tracing-test", shouldTrace: () => { throw new Error("predicate failed"); @@ -221,15 +226,10 @@ describe("traceAsyncChannel current span binding", () => { extractMetrics: () => ({}), }); - try { - await testChannels.asyncCall.tracePromise( - async () => ({ ok: true as const }), - { arguments: [{}] } as any, - ); - } finally { - unsubscribe(); - } - + await testChannels.throwingPredicate.tracePromise( + async () => ({ ok: true as const }), + { arguments: [{}] } as any, + ); expect(consoleErrorSpy).not.toHaveBeenCalled(); consoleErrorSpy.mockRestore(); @@ -238,7 +238,7 @@ describe("traceAsyncChannel current span binding", () => { }); it("skips auto instrumentation spans while suppression is active", async () => { - const unsubscribe = traceAsyncChannel(testChannels.asyncCall, { + traceAsyncChannel(testChannels.suppressed, { name: "channel-tracing-test", type: "function", extractInput: () => ({ @@ -249,23 +249,18 @@ describe("traceAsyncChannel current span binding", () => { extractMetrics: () => ({}), }); - try { - await runWithAutoInstrumentationSuppressed(() => - testChannels.asyncCall.tracePromise( - async () => { - expect(currentSpan()).toBe(NOOP_SPAN); - await Promise.resolve(); - expect(currentSpan()).toBe(NOOP_SPAN); - - return { ok: true as const }; - }, - { arguments: [{}] } as any, - ), - ); - } finally { - unsubscribe(); - } + await runWithAutoInstrumentationSuppressed(() => + testChannels.suppressed.tracePromise( + async () => { + expect(currentSpan()).toBe(NOOP_SPAN); + await Promise.resolve(); + expect(currentSpan()).toBe(NOOP_SPAN); + return { ok: true as const }; + }, + { arguments: [{}] } as any, + ), + ); const spans = await backgroundLogger.drain(); expect(spans).toHaveLength(0); }); @@ -280,7 +275,7 @@ describe("traceAsyncChannel current span binding", () => { throw new Error("logging failed"); }), } as unknown as Span; - const unsubscribe = traceStreamingChannel(testChannels.streamingCall, { + traceStreamingChannel(testChannels.streamingCleanup, { name: "streaming-channel-test", startSpan: () => child, type: "function", @@ -291,25 +286,20 @@ describe("traceAsyncChannel current span binding", () => { onError, }); - try { - await expect( - testChannels.streamingCall.tracePromise( - async () => ({ ok: true as const }), - { arguments: [{}] } as any, - ), - ).resolves.toEqual({ ok: true }); - await expect( - testChannels.streamingCall.tracePromise( - async () => { - throw new Error("call failed"); - }, - { arguments: [{}] } as any, - ), - ).rejects.toThrow("call failed"); - } finally { - unsubscribe(); - } - + await expect( + testChannels.streamingCleanup.tracePromise( + async () => ({ ok: true as const }), + { arguments: [{}] } as any, + ), + ).resolves.toEqual({ ok: true }); + await expect( + testChannels.streamingCleanup.tracePromise( + async () => { + throw new Error("call failed"); + }, + { arguments: [{}] } as any, + ), + ).rejects.toThrow("call failed"); expect(onComplete).toHaveBeenCalledTimes(1); expect(onError).toHaveBeenCalledTimes(1); expect(end).toHaveBeenCalledTimes(2); @@ -321,7 +311,7 @@ describe("traceAsyncChannel current span binding", () => { end: vi.fn(), log: vi.fn(), } as unknown as Span; - const unsubscribe = traceStreamingChannel(testChannels.streamingCall, { + traceStreamingChannel(testChannels.streamingCancellation, { name: "streaming-channel-test", startSpan: () => child, type: "function", @@ -337,17 +327,12 @@ describe("traceAsyncChannel current span binding", () => { }, }; - try { - const patched = await testChannels.streamingCall.tracePromise( - async () => stream as any, - { arguments: [{}] } as any, - ); - (patched as unknown as typeof stream).abort(); - await Promise.resolve(); - } finally { - unsubscribe(); - } - + const patched = await testChannels.streamingCancellation.tracePromise( + async () => stream as any, + { arguments: [{}] } as any, + ); + (patched as unknown as typeof stream).abort(); + await Promise.resolve(); const cancellationError = expect.objectContaining({ message: "Stream cancelled before completion", name: "AbortError", diff --git a/js/src/instrumentation/core/channel-tracing.ts b/js/src/instrumentation/core/channel-tracing.ts index a8ecc819c..1f3ff47d0 100644 --- a/js/src/instrumentation/core/channel-tracing.ts +++ b/js/src/instrumentation/core/channel-tracing.ts @@ -312,7 +312,7 @@ function bindCurrentSpanStoreToStart< }, channelName: string, instrumentationName: SpanInstrumentationName, -): (() => void) | undefined { +): void { const state = _internalGetGlobalState(); const startChannel = tracingChannel.start; const contextManager = state?.contextManager; @@ -325,7 +325,7 @@ function bindCurrentSpanStoreToStart< : undefined; if (!currentSpanStore || !startChannel) { - return undefined; + return; } startChannel.bindStore( @@ -347,10 +347,6 @@ function bindCurrentSpanStoreToStart< : currentSpanStore.getStore(); }, ); - - return () => { - startChannel.unbindStore(currentSpanStore); - }; } function logErrorAndEnd< @@ -442,13 +438,13 @@ function runStreamingErrorHook(args: { export function traceAsyncChannel( channel: TChannel, config: AsyncChannelSpanConfig, -): () => void { +): void { const tracingChannel = channel.tracingChannel() as IsoTracingChannel< ChannelMessage >; const states = new WeakMap(); const channelName = channel.channelName; - const unbindCurrentSpanStore = bindCurrentSpanStoreToStart( + bindCurrentSpanStoreToStart( tracingChannel, states, config, @@ -514,23 +510,18 @@ export function traceAsyncChannel( }; tracingChannel.subscribe(handlers); - - return () => { - unbindCurrentSpanStore?.(); - tracingChannel.unsubscribe(handlers); - }; } export function traceStreamingChannel( channel: TChannel, config: StreamingChannelSpanConfig, -): () => void { +): void { const tracingChannel = channel.tracingChannel() as IsoTracingChannel< ChannelMessage >; const states = new WeakMap(); const channelName = channel.channelName; - const unbindCurrentSpanStore = bindCurrentSpanStoreToStart( + bindCurrentSpanStoreToStart( tracingChannel, states, config, @@ -790,23 +781,18 @@ export function traceStreamingChannel( }; tracingChannel.subscribe(handlers); - - return () => { - unbindCurrentSpanStore?.(); - tracingChannel.unsubscribe(handlers); - }; } export function traceSyncStreamChannel( channel: TChannel, config: SyncStreamChannelSpanConfig, -): () => void { +): void { const tracingChannel = channel.tracingChannel() as IsoTracingChannel< ChannelMessage >; const states = new WeakMap(); const channelName = channel.channelName; - const unbindCurrentSpanStore = bindCurrentSpanStoreToStart( + bindCurrentSpanStoreToStart( tracingChannel, states, config, @@ -939,19 +925,4 @@ export function traceSyncStreamChannel( }; tracingChannel.subscribe(handlers); - - return () => { - unbindCurrentSpanStore?.(); - tracingChannel.unsubscribe(handlers); - }; -} - -export function unsubscribeAll( - unsubscribers: Array<() => void>, -): Array<() => void> { - for (const unsubscribe of unsubscribers) { - unsubscribe(); - } - - return []; } diff --git a/js/src/instrumentation/core/channel.ts b/js/src/instrumentation/core/channel.ts deleted file mode 100644 index 8cbf253d6..000000000 --- a/js/src/instrumentation/core/channel.ts +++ /dev/null @@ -1,55 +0,0 @@ -/** - * Utilities for instrumentation hook naming and management. - */ - -/** - * Channel naming convention: braintrust:{component}:{operation} - * - * Examples: - * - braintrust:openai:chat.completions.create - * - braintrust:anthropic:messages.create - * - braintrust:ai-sdk:generateText - */ - -/** - * Creates a standardized channel name. - * - * @param component - The SDK/library being instrumented (e.g., 'openai', 'anthropic') - * @param operation - The operation being traced (e.g., 'chat.completions.create') - * @returns The full channel name - */ -export function createChannelName( - component: string, - operation: string, -): string { - return `braintrust:${component}:${operation}`; -} - -/** - * Parses a channel name into its component parts. - * - * @param channelName - The full channel name - * @returns Object with component and operation, or null if invalid - */ -export function parseChannelName( - channelName: string, -): { component: string; operation: string } | null { - const match = channelName.match(/^braintrust:([^:]+):(.+)$/); - if (!match) { - return null; - } - return { - component: match[1], - operation: match[2], - }; -} - -/** - * Validates a channel name follows the expected convention. - * - * @param channelName - The channel name to validate - * @returns True if valid - */ -export function isValidChannelName(channelName: string): boolean { - return /^braintrust:[^:]+:.+$/.test(channelName); -} diff --git a/js/src/instrumentation/core/index.ts b/js/src/instrumentation/core/index.ts index 6e2d2d61c..edfe17a89 100644 --- a/js/src/instrumentation/core/index.ts +++ b/js/src/instrumentation/core/index.ts @@ -1,27 +1,3 @@ -/** - * Core utilities for building auto-instrumentation plugins. - * - * Provides BasePlugin class and channel utilities following the OpenTelemetry - * InstrumentationBase pattern - core infrastructure lives here, but individual - * instrumentations can be separate packages. - * - * Note: auto-instrumentation config types are exposed from the Braintrust - * bundler subpaths, such as `braintrust/vite`. - */ +/** Internal utilities shared by Braintrust instrumentation consumers. */ -export { BasePlugin } from "./plugin"; export { toLoggedError } from "./logging"; -export { - createChannelName, - parseChannelName, - isValidChannelName, -} from "./channel"; -export type { - BaseContext, - StartEvent, - EndEvent, - ErrorEvent, - AsyncStartEvent, - AsyncEndEvent, - ChannelHandlers, -} from "./types"; diff --git a/js/src/instrumentation/core/plugin.ts b/js/src/instrumentation/core/plugin.ts deleted file mode 100644 index ae09a43f1..000000000 --- a/js/src/instrumentation/core/plugin.ts +++ /dev/null @@ -1,505 +0,0 @@ -import iso from "../../isomorph"; -import type { IsoChannelHandlers } from "../../isomorph"; -import { isAsyncIterable, patchStreamIfNeeded } from "./stream-patcher"; -import type { StartEvent } from "./types"; -import { startSpan } from "../../logger"; -import type { Span } from "../../logger"; -import { getCurrentUnixTimestamp } from "../../util"; -import { - buildStartSpanArgs, - mergeInputMetadata, -} from "./channel-tracing-utils"; - -/** - * Base class for creating instrumentation plugins. - * - * Plugins subscribe to global instrumentation hook events and convert them - * into spans, logs, or other observability data. - */ -export abstract class BasePlugin { - protected enabled = false; - protected unsubscribers: Array<() => void> = []; - - /** - * Enables the plugin. Must be called before the plugin will receive events. - */ - enable(): void { - if (this.enabled) { - return; - } - this.enabled = true; - this.onEnable(); - } - - /** - * Disables the plugin. After this, the plugin will no longer receive events. - */ - disable(): void { - if (!this.enabled) { - return; - } - this.enabled = false; - this.onDisable(); - } - - /** - * Called when the plugin is enabled. - * Override this to set up subscriptions. - */ - protected abstract onEnable(): void; - - /** - * Called when the plugin is disabled. - * Override this to clean up subscriptions. - */ - protected abstract onDisable(): void; - - /** - * Helper to subscribe to a channel with raw handlers. - * - * @param channelName - The channel name to subscribe to - * @param handlers - Event handlers - */ - protected subscribe(channelName: string, handlers: IsoChannelHandlers): void { - const channel = iso.newTracingChannel(channelName); - channel.subscribe(handlers); - } - - /** - * Subscribe to a channel for async methods (non-streaming). - * Creates a span and logs input/output/metrics. - */ - protected subscribeToChannel( - channelName: string, - config: { - name: string; - type: string; - extractInput: (args: any[]) => { input: any; metadata: any }; - extractOutput: (result: any, endEvent?: any) => any; - extractMetadata?: (result: any, endEvent?: any) => any; - extractMetrics: ( - result: any, - startTime?: number, - endEvent?: any, - ) => Record; - }, - ): void { - const channel = iso.newTracingChannel(channelName); - - const spans = new WeakMap(); - - const handlers = { - start: (event: StartEvent) => { - const { name, spanAttributes, spanInfoMetadata } = buildStartSpanArgs( - config, - event, - ); - const span = startSpan({ - name, - spanAttributes, - }); - - const startTime = getCurrentUnixTimestamp(); - spans.set(event, { span, startTime }); - - try { - const { input, metadata } = config.extractInput(event.arguments); - span.log({ - input, - metadata: mergeInputMetadata(metadata, spanInfoMetadata), - }); - } catch (error) { - // eslint-disable-next-line no-restricted-properties -- preserving intentional console usage. - console.error(`Error extracting input for ${channelName}:`, error); - } - }, - - asyncEnd: (event: any) => { - const spanData = spans.get(event); - if (!spanData) { - return; - } - - const { span, startTime } = spanData; - - try { - const output = config.extractOutput(event.result, event); - const metrics = config.extractMetrics(event.result, startTime, event); - const metadata = config.extractMetadata?.(event.result, event); - - span.log({ - output, - ...(metadata !== undefined ? { metadata } : {}), - metrics, - }); - } catch (error) { - // eslint-disable-next-line no-restricted-properties -- preserving intentional console usage. - console.error(`Error extracting output for ${channelName}:`, error); - } finally { - span.end(); - spans.delete(event); - } - }, - - error: (event: any) => { - const spanData = spans.get(event); - if (!spanData) { - return; - } - - const { span } = spanData; - - span.log({ - error: event.error.message, - }); - span.end(); - spans.delete(event); - }, - }; - - channel.subscribe(handlers); - - // Store unsubscribe function - this.unsubscribers.push(() => { - channel.unsubscribe(handlers); - }); - } - - /** - * Subscribe to a channel for async methods that may return streams. - * Handles both streaming and non-streaming responses. - */ - protected subscribeToStreamingChannel( - channelName: string, - config: { - name: string; - type: string; - extractInput: (args: any[]) => { input: any; metadata: any }; - extractOutput: (result: any, endEvent?: any) => any; - extractMetadata?: (result: any, endEvent?: any) => any; - extractMetrics: ( - result: any, - startTime?: number, - endEvent?: any, - ) => Record; - aggregateChunks?: ( - chunks: any[], - result?: any, - endEvent?: any, - ) => { - output: any; - metrics: Record; - metadata?: any; - }; - }, - ): void { - const channel = iso.newTracingChannel(channelName); - - const spans = new WeakMap(); - - const handlers = { - start: (event: StartEvent) => { - const { name, spanAttributes, spanInfoMetadata } = buildStartSpanArgs( - config, - event, - ); - const span = startSpan({ - name, - spanAttributes, - }); - - const startTime = getCurrentUnixTimestamp(); - spans.set(event, { span, startTime }); - - try { - const { input, metadata } = config.extractInput(event.arguments); - span.log({ - input, - metadata: mergeInputMetadata(metadata, spanInfoMetadata), - }); - } catch (error) { - // eslint-disable-next-line no-restricted-properties -- preserving intentional console usage. - console.error(`Error extracting input for ${channelName}:`, error); - } - }, - - asyncEnd: (event: any) => { - const spanData = spans.get(event); - if (!spanData) { - return; - } - - const { span, startTime } = spanData; - - // Check if result is a stream - if (isAsyncIterable(event.result)) { - let firstChunkTime: number | undefined; - - // Patch the stream to collect chunks - patchStreamIfNeeded(event.result, { - onChunk: () => { - if (firstChunkTime === undefined) { - firstChunkTime = getCurrentUnixTimestamp(); - } - }, - onComplete: (chunks: any[]) => { - try { - let output: any; - let metrics: Record; - let metadata: any; - - if (config.aggregateChunks) { - const aggregated = config.aggregateChunks( - chunks, - event.result, - event, - ); - output = aggregated.output; - metrics = aggregated.metrics; - metadata = aggregated.metadata; - } else { - output = config.extractOutput(chunks, event); - metrics = config.extractMetrics(chunks, startTime, event); - } - - // Add time_to_first_token if not already present - if ( - metrics.time_to_first_token === undefined && - firstChunkTime !== undefined - ) { - metrics.time_to_first_token = firstChunkTime - startTime; - } else if ( - metrics.time_to_first_token === undefined && - chunks.length > 0 - ) { - metrics.time_to_first_token = - getCurrentUnixTimestamp() - startTime; - } - - span.log({ - output, - ...(metadata !== undefined ? { metadata } : {}), - metrics, - }); - } catch (error) { - // eslint-disable-next-line no-restricted-properties -- preserving intentional console usage. - console.error( - `Error extracting output for ${channelName}:`, - error, - ); - } finally { - span.end(); - } - }, - onError: (error: Error) => { - span.log({ - error: error.message, - }); - span.end(); - }, - }); - - // Don't delete the span from the map yet - it will be ended by the stream - } else { - // Non-streaming response - try { - const output = config.extractOutput(event.result, event); - const metadata = config.extractMetadata - ? config.extractMetadata(event.result, event) - : undefined; - const metrics = config.extractMetrics( - event.result, - startTime, - event, - ); - - span.log({ - output, - ...(metadata !== undefined ? { metadata } : {}), - metrics, - }); - } catch (error) { - // eslint-disable-next-line no-restricted-properties -- preserving intentional console usage. - console.error(`Error extracting output for ${channelName}:`, error); - } finally { - span.end(); - spans.delete(event); - } - } - }, - - error: (event: any) => { - const spanData = spans.get(event); - if (!spanData) { - return; - } - - const { span } = spanData; - - span.log({ - error: event.error.message, - }); - span.end(); - spans.delete(event); - }, - }; - - channel.subscribe(handlers); - - // Store unsubscribe function - this.unsubscribers.push(() => { - channel.unsubscribe(handlers); - }); - } - - /** - * Subscribe to a channel for sync methods that return event-based streams. - * Used for methods like beta.chat.completions.stream() and responses.stream(). - */ - protected subscribeToSyncStreamChannel( - channelName: string, - config: { - name: string; - type: string; - extractInput: (args: any[]) => { input: any; metadata: any }; - extractFromEvent?: (event: any) => { - output?: any; - metrics?: Record; - metadata?: any; - }; - }, - ): void { - const channel = iso.newTracingChannel(channelName); - - const spans = new WeakMap(); - - const handlers = { - start: (event: StartEvent) => { - const { name, spanAttributes, spanInfoMetadata } = buildStartSpanArgs( - config, - event, - ); - const span = startSpan({ - name, - spanAttributes, - }); - - const startTime = getCurrentUnixTimestamp(); - spans.set(event, { span, startTime }); - - try { - const { input, metadata } = config.extractInput(event.arguments); - span.log({ - input, - metadata: mergeInputMetadata(metadata, spanInfoMetadata), - }); - } catch (error) { - // eslint-disable-next-line no-restricted-properties -- preserving intentional console usage. - console.error(`Error extracting input for ${channelName}:`, error); - } - }, - - end: (event: any) => { - const spanData = spans.get(event); - if (!spanData) { - return; - } - - const { span, startTime } = spanData; - const stream = event.result; - - if (!stream || typeof stream.on !== "function") { - // Not a stream, just end the span - span.end(); - spans.delete(event); - return; - } - - let first = true; - - // Listen for stream events - stream.on("chunk", (chunk: any) => { - if (first) { - const now = getCurrentUnixTimestamp(); - span.log({ - metrics: { - time_to_first_token: now - startTime, - }, - }); - first = false; - } - }); - - stream.on("chatCompletion", (completion: any) => { - try { - span.log({ - output: completion.choices, - }); - } catch (error) { - // eslint-disable-next-line no-restricted-properties -- preserving intentional console usage. - console.error( - `Error extracting chatCompletion for ${channelName}:`, - error, - ); - } - }); - - stream.on("event", (streamEvent: any) => { - if (config.extractFromEvent) { - try { - if (first) { - const now = getCurrentUnixTimestamp(); - span.log({ - metrics: { - time_to_first_token: now - startTime, - }, - }); - first = false; - } - - const extracted = config.extractFromEvent(streamEvent); - if (extracted && Object.keys(extracted).length > 0) { - span.log(extracted); - } - } catch (error) { - // eslint-disable-next-line no-restricted-properties -- preserving intentional console usage. - console.error( - `Error extracting event for ${channelName}:`, - error, - ); - } - } - }); - - stream.on("end", () => { - span.end(); - spans.delete(event); - }); - - // Don't delete the span from the map - it will be deleted when the stream ends - }, - - error: (event: any) => { - const spanData = spans.get(event); - if (!spanData) { - return; - } - - const { span } = spanData; - - span.log({ - error: event.error.message, - }); - span.end(); - spans.delete(event); - }, - }; - - channel.subscribe(handlers); - - // Store unsubscribe function - this.unsubscribers.push(() => { - channel.unsubscribe(handlers); - }); - } -} diff --git a/js/src/instrumentation/core/types.ts b/js/src/instrumentation/core/types.ts index dc2434dc3..0181f3196 100644 --- a/js/src/instrumentation/core/types.ts +++ b/js/src/instrumentation/core/types.ts @@ -27,7 +27,7 @@ export type SpanInfoCarrier< /** * Base context object shared across all events in a trace. */ -export interface BaseContext { +interface BaseContext { /** * Unique identifier for this trace. * Can be used to correlate start/end/error events. @@ -52,42 +52,6 @@ export interface StartEvent extends BaseContext { arguments: TInput[]; } -/** - * Event emitted after the synchronous portion completes. - * For async functions, this fires when the promise is returned (not settled). - */ -export interface EndEvent extends BaseContext { - /** - * The result of the synchronous portion. - * For async functions, this is the promise (not the resolved value). - */ - result: TResult; - - /** - * Arguments passed to the function (also available in StartEvent). - */ - arguments?: unknown[]; -} - -/** - * Event emitted when a function throws or a promise rejects. - */ -export interface ErrorEvent extends BaseContext { - /** - * The error that was thrown or the rejection reason. - */ - error: Error; - - /** - * Arguments passed to the function (also available in StartEvent). - */ - arguments?: unknown[]; -} - -/** - * Event emitted when a promise begins to settle. - * This fires after the synchronous portion and when the async continuation starts. - */ export interface TypedStartEvent< TArguments extends EventArguments = unknown[], > extends BaseContext { @@ -112,17 +76,6 @@ export interface TypedErrorEvent< arguments?: [...TArguments, ...any[]]; } -// eslint-disable-next-line @typescript-eslint/no-empty-object-type -export interface AsyncStartEvent extends StartEvent {} - -/** - * Event emitted when a promise finishes settling. - * This fires BEFORE control returns to user code after await. - * This is where you should extract output data and finalize spans. - */ -// eslint-disable-next-line @typescript-eslint/no-empty-object-type -export interface AsyncEndEvent extends EndEvent {} - export type StartEventWith< TArguments extends EventArguments = unknown[], TExtra extends object = Record, @@ -144,43 +97,3 @@ export type ErrorEventWith< TArguments extends EventArguments = unknown[], TExtra extends object = Record, > = TypedErrorEvent & TExtra; - -/** - * Subscription handlers for a tracing-compatible global hook. - * - * Common usage pattern: - * - Use start to create spans and extract input - * - Use asyncEnd to extract output and finalize spans - * - Use error to handle failures - */ -export interface ChannelHandlers { - /** - * Called before the synchronous portion of a function executes. - * Use this to create spans and extract input data. - */ - start?: (event: StartEvent) => void; - - /** - * Called after the synchronous portion completes (promise returned). - * Usually not needed for typical instrumentation. - */ - end?: (event: EndEvent) => void; - - /** - * Called when a promise begins to settle. - * Usually not needed for typical instrumentation. - */ - asyncStart?: (event: AsyncStartEvent) => void; - - /** - * Called when a promise finishes settling, before user code continues. - * Use this to extract output, patch streams, and finalize spans. - */ - asyncEnd?: (event: AsyncEndEvent) => void; - - /** - * Called when a function throws or promise rejects. - * Use this to log errors and clean up spans. - */ - error?: (event: ErrorEvent) => void; -} diff --git a/js/src/instrumentation/index.test.ts b/js/src/instrumentation/index.test.ts new file mode 100644 index 000000000..fd5442745 --- /dev/null +++ b/js/src/instrumentation/index.test.ts @@ -0,0 +1,13 @@ +import { describe, expect, it } from "vitest"; +import * as instrumentation from "./index"; + +describe("instrumentation public API", () => { + it("exposes only the curated instrumentation APIs", () => { + expect(Object.keys(instrumentation).sort()).toEqual([ + "OpenAIAgentsTraceProcessor", + "braintrustEveInstrumentation", + "braintrustFlueInstrumentation", + "configureInstrumentation", + ]); + }); +}); diff --git a/js/src/instrumentation/index.ts b/js/src/instrumentation/index.ts index 8bbbd31ec..e148f2dc5 100644 --- a/js/src/instrumentation/index.ts +++ b/js/src/instrumentation/index.ts @@ -1,42 +1,16 @@ /** * Instrumentation APIs for auto-instrumentation. * - * This module provides the core plugin infrastructure for converting global - * instrumentation hook events into Braintrust spans. - * - * Following the OpenTelemetry pattern, BasePlugin (like InstrumentationBase) - * lives in the core SDK, while individual instrumentation implementations - * can be separate packages. - * * For auto-instrumentation config types, import InstrumentationConfig from the * relevant Braintrust bundler subpath such as `braintrust/vite`. * * @module instrumentation */ -export { BasePlugin } from "./core"; -export { BraintrustPlugin } from "./braintrust-plugin"; -export type { BraintrustPluginConfig } from "./braintrust-plugin"; -export { OpenAIAgentsTraceProcessor } from "./plugins/openai-agents-trace-processor"; -export type { OpenAIAgentsTraceProcessorOptions } from "./plugins/openai-agents-trace-processor"; -export { braintrustFlueInstrumentation } from "./plugins/flue-plugin"; -export { braintrustEveInstrumentation } from "./plugins/eve-plugin"; - -// Re-export core types for external instrumentation packages -export type { - BaseContext, - StartEvent, - EndEvent, - ErrorEvent, - AsyncStartEvent, - AsyncEndEvent, - ChannelHandlers, -} from "./core"; -export { - createChannelName, - parseChannelName, - isValidChannelName, -} from "./core"; +export { OpenAIAgentsTraceProcessor } from "./providers/openai-agents-trace-processor"; +export type { OpenAIAgentsTraceProcessorOptions } from "./providers/openai-agents-trace-processor"; +export { braintrustFlueInstrumentation } from "./providers/flue-instrumentation"; +export { braintrustEveInstrumentation } from "./providers/eve-instrumentation"; // Configuration API export { configureInstrumentation } from "./registry"; diff --git a/js/src/instrumentation/instrumentation-consumers.test.ts b/js/src/instrumentation/instrumentation-consumers.test.ts new file mode 100644 index 000000000..ce4837bb5 --- /dev/null +++ b/js/src/instrumentation/instrumentation-consumers.test.ts @@ -0,0 +1,215 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; +import type { InstrumentationIntegrationsConfig } from "./config"; + +const registrations = vi.hoisted(() => ({ + aiSDK: vi.fn(), + anthropic: vi.fn(), + bedrockRuntime: vi.fn(), + claudeAgentSDK: vi.fn(), + cloudflareAgents: vi.fn(), + cloudflareAIChat: vi.fn(), + cloudflareThink: vi.fn(), + cohere: vi.fn(), + cursorSDK: vi.fn(), + genkit: vi.fn(), + gitHubCopilot: vi.fn(), + googleADK: vi.fn(), + googleGenAI: vi.fn(), + groq: vi.fn(), + huggingFace: vi.fn(), + huggingFaceTransformers: vi.fn(), + langChain: vi.fn(), + langSmith: vi.fn(), + mistral: vi.fn(), + ollama: vi.fn(), + openAI: vi.fn(), + openAIAgents: vi.fn(), + openAICodex: vi.fn(), + openRouter: vi.fn(), + openRouterAgent: vi.fn(), + piCodingAgent: vi.fn(), + strandsAgentSDK: vi.fn(), + voyageAI: vi.fn(), +})); + +vi.mock("./providers/ai-sdk-instrumentation", () => ({ + registerAISDKInstrumentation: registrations.aiSDK, +})); +vi.mock("./providers/anthropic-instrumentation", () => ({ + registerAnthropicInstrumentation: registrations.anthropic, +})); +vi.mock("./providers/bedrock-runtime-instrumentation", () => ({ + registerBedrockRuntimeInstrumentation: registrations.bedrockRuntime, +})); +vi.mock("./providers/claude-agent-sdk-instrumentation", () => ({ + registerClaudeAgentSDKInstrumentation: registrations.claudeAgentSDK, +})); +vi.mock("./providers/cloudflare-agents-instrumentation", () => ({ + registerCloudflareAgentsInstrumentation: registrations.cloudflareAgents, +})); +vi.mock("./providers/cloudflare-ai-chat-consumer", () => ({ + registerCloudflareAIChatInstrumentation: registrations.cloudflareAIChat, +})); +vi.mock("./providers/cloudflare-think-instrumentation", () => ({ + registerCloudflareThinkInstrumentation: registrations.cloudflareThink, +})); +vi.mock("./providers/cohere-instrumentation", () => ({ + registerCohereInstrumentation: registrations.cohere, +})); +vi.mock("./providers/cursor-sdk-instrumentation", () => ({ + registerCursorSDKInstrumentation: registrations.cursorSDK, +})); +vi.mock("./providers/genkit-instrumentation", () => ({ + registerGenkitInstrumentation: registrations.genkit, +})); +vi.mock("./providers/github-copilot-instrumentation", () => ({ + registerGitHubCopilotInstrumentation: registrations.gitHubCopilot, +})); +vi.mock("./providers/google-adk-instrumentation", () => ({ + registerGoogleADKInstrumentation: registrations.googleADK, +})); +vi.mock("./providers/google-genai-instrumentation", () => ({ + registerGoogleGenAIInstrumentation: registrations.googleGenAI, +})); +vi.mock("./providers/groq-instrumentation", () => ({ + registerGroqInstrumentation: registrations.groq, +})); +vi.mock("./providers/huggingface-instrumentation", () => ({ + registerHuggingFaceInstrumentation: registrations.huggingFace, +})); +vi.mock("./providers/huggingface-transformers-instrumentation", () => ({ + registerHuggingFaceTransformersInstrumentation: + registrations.huggingFaceTransformers, +})); +vi.mock("./providers/langchain-instrumentation", () => ({ + registerLangChainInstrumentation: registrations.langChain, +})); +vi.mock("./providers/langsmith-instrumentation", () => ({ + registerLangSmithInstrumentation: registrations.langSmith, +})); +vi.mock("./providers/mistral-instrumentation", () => ({ + registerMistralInstrumentation: registrations.mistral, +})); +vi.mock("./providers/ollama-instrumentation", () => ({ + registerOllamaInstrumentation: registrations.ollama, +})); +vi.mock("./providers/openai-instrumentation", () => ({ + registerOpenAIInstrumentation: registrations.openAI, +})); +vi.mock("./providers/openai-agents-instrumentation", () => ({ + registerOpenAIAgentsInstrumentation: registrations.openAIAgents, +})); +vi.mock("./providers/openai-codex-instrumentation", () => ({ + registerOpenAICodexInstrumentation: registrations.openAICodex, +})); +vi.mock("./providers/openrouter-instrumentation", () => ({ + registerOpenRouterInstrumentation: registrations.openRouter, +})); +vi.mock("./providers/openrouter-agent-instrumentation", () => ({ + registerOpenRouterAgentInstrumentation: registrations.openRouterAgent, +})); +vi.mock("./providers/pi-coding-agent-instrumentation", () => ({ + registerPiCodingAgentInstrumentation: registrations.piCodingAgent, +})); +vi.mock("./providers/strands-agent-sdk-instrumentation", () => ({ + registerStrandsAgentSDKInstrumentation: registrations.strandsAgentSDK, +})); +vi.mock("./providers/voyageai-instrumentation", () => ({ + registerVoyageAIInstrumentation: registrations.voyageAI, +})); + +import { registerInstrumentationConsumers } from "./instrumentation-consumers"; + +const integrationCases: Array< + [ + keyof InstrumentationIntegrationsConfig, + Array<(typeof registrations)[keyof typeof registrations]>, + ] +> = [ + ["aisdk", [registrations.aiSDK]], + ["anthropic", [registrations.anthropic]], + ["bedrock", [registrations.bedrockRuntime]], + ["claudeAgentSDK", [registrations.claudeAgentSDK]], + ["cloudflareAgents", [registrations.cloudflareAgents]], + ["cloudflareAIChat", [registrations.cloudflareAIChat]], + ["cloudflareThink", [registrations.cloudflareThink]], + ["cohere", [registrations.cohere]], + ["cursorSDK", [registrations.cursorSDK]], + ["genkit", [registrations.genkit]], + ["gitHubCopilot", [registrations.gitHubCopilot]], + ["googleADK", [registrations.googleADK]], + ["googleGenAI", [registrations.googleGenAI]], + ["groq", [registrations.groq]], + [ + "huggingface", + [registrations.huggingFace, registrations.huggingFaceTransformers], + ], + ["langchain", [registrations.langChain]], + ["langsmith", [registrations.langSmith]], + ["mistral", [registrations.mistral]], + ["ollama", [registrations.ollama]], + ["openai", [registrations.openAI]], + ["openAIAgents", [registrations.openAIAgents]], + ["openaiCodexSDK", [registrations.openAICodex]], + ["openrouter", [registrations.openRouter]], + ["openrouterAgent", [registrations.openRouterAgent]], + ["piCodingAgent", [registrations.piCodingAgent]], + ["strandsAgentSDK", [registrations.strandsAgentSDK]], + ["voyageai", [registrations.voyageAI]], +]; + +describe("registerInstrumentationConsumers", () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + it("registers every instrumentation consumer by default", () => { + registerInstrumentationConsumers(); + + for (const register of Object.values(registrations)) { + expect(register).toHaveBeenCalledTimes(1); + } + expect(registrations.langSmith).toHaveBeenCalledWith({ + skipLangChainRuns: true, + }); + }); + + it.each(integrationCases)( + "does not register the %s integration when disabled", + (integration, disabledRegistrations) => { + registerInstrumentationConsumers({ + integrations: { [integration]: false }, + }); + + for (const register of disabledRegistrations) { + expect(register).not.toHaveBeenCalled(); + } + }, + ); + + it.each([ + ["vercel", registrations.aiSDK], + ["google", registrations.googleGenAI], + ["cursor", registrations.cursorSDK], + ["awsBedrock", registrations.bedrockRuntime], + ["awsBedrockRuntime", registrations.bedrockRuntime], + ["langgraph", registrations.langChain], + ] as const)( + "honors the legacy or related %s disable flag", + (integration, register) => { + registerInstrumentationConsumers({ + integrations: { [integration]: false }, + }); + + expect(register).not.toHaveBeenCalled(); + }, + ); + + it("tells LangSmith not to suppress LangChain runs when LangChain is disabled", () => { + registerInstrumentationConsumers({ integrations: { langchain: false } }); + + expect(registrations.langSmith).toHaveBeenCalledWith({ + skipLangChainRuns: false, + }); + }); +}); diff --git a/js/src/instrumentation/instrumentation-consumers.ts b/js/src/instrumentation/instrumentation-consumers.ts new file mode 100644 index 000000000..ed32522a9 --- /dev/null +++ b/js/src/instrumentation/instrumentation-consumers.ts @@ -0,0 +1,159 @@ +import { registerOpenAIInstrumentation } from "./providers/openai-instrumentation"; +import { registerOpenAICodexInstrumentation } from "./providers/openai-codex-instrumentation"; +import { registerAnthropicInstrumentation } from "./providers/anthropic-instrumentation"; +import { registerAISDKInstrumentation } from "./providers/ai-sdk-instrumentation"; +import { registerClaudeAgentSDKInstrumentation } from "./providers/claude-agent-sdk-instrumentation"; +import { registerCloudflareThinkInstrumentation } from "./providers/cloudflare-think-instrumentation"; +import { registerCursorSDKInstrumentation } from "./providers/cursor-sdk-instrumentation"; +import { registerOpenAIAgentsInstrumentation } from "./providers/openai-agents-instrumentation"; +import { registerGoogleGenAIInstrumentation } from "./providers/google-genai-instrumentation"; +import { registerHuggingFaceInstrumentation } from "./providers/huggingface-instrumentation"; +import { registerHuggingFaceTransformersInstrumentation } from "./providers/huggingface-transformers-instrumentation"; +import { registerOpenRouterAgentInstrumentation } from "./providers/openrouter-agent-instrumentation"; +import { registerOpenRouterInstrumentation } from "./providers/openrouter-instrumentation"; +import { registerMistralInstrumentation } from "./providers/mistral-instrumentation"; +import { registerOllamaInstrumentation } from "./providers/ollama-instrumentation"; +import { registerGoogleADKInstrumentation } from "./providers/google-adk-instrumentation"; +import { registerCohereInstrumentation } from "./providers/cohere-instrumentation"; +import { registerGroqInstrumentation } from "./providers/groq-instrumentation"; +import { registerBedrockRuntimeInstrumentation } from "./providers/bedrock-runtime-instrumentation"; +import { registerGenkitInstrumentation } from "./providers/genkit-instrumentation"; +import { registerGitHubCopilotInstrumentation } from "./providers/github-copilot-instrumentation"; +import { registerLangChainInstrumentation } from "./providers/langchain-instrumentation"; +import { registerLangSmithInstrumentation } from "./providers/langsmith-instrumentation"; +import { registerPiCodingAgentInstrumentation } from "./providers/pi-coding-agent-instrumentation"; +import { registerStrandsAgentSDKInstrumentation } from "./providers/strands-agent-sdk-instrumentation"; +import { registerVoyageAIInstrumentation } from "./providers/voyageai-instrumentation"; +import { registerCloudflareAIChatInstrumentation } from "./providers/cloudflare-ai-chat-consumer"; +import { registerCloudflareAgentsInstrumentation } from "./providers/cloudflare-agents-instrumentation"; +import type { InstrumentationConfig } from "./config"; + +/** Registers the configured process-lifetime instrumentation consumers. */ +export function registerInstrumentationConsumers( + config: InstrumentationConfig = {}, +): void { + const integrations = config.integrations ?? {}; + + if (integrations.openai !== false) { + registerOpenAIInstrumentation(); + } + + if (integrations.openaiCodexSDK !== false) { + registerOpenAICodexInstrumentation(); + } + + if (integrations.anthropic !== false) { + registerAnthropicInstrumentation(); + } + + // Support both 'aisdk' and legacy 'vercel' config keys + if (integrations.aisdk !== false && integrations.vercel !== false) { + registerAISDKInstrumentation(); + } + + if (integrations.claudeAgentSDK !== false) { + registerClaudeAgentSDKInstrumentation(); + } + + if (integrations.cloudflareThink !== false) { + registerCloudflareThinkInstrumentation(); + } + + if (integrations.cursorSDK !== false && integrations.cursor !== false) { + registerCursorSDKInstrumentation(); + } + + if (integrations.openAIAgents !== false) { + registerOpenAIAgentsInstrumentation(); + } + + // Support both 'googleGenAI' and legacy 'google' config keys + if (integrations.googleGenAI !== false && integrations.google !== false) { + registerGoogleGenAIInstrumentation(); + } + + if (integrations.huggingface !== false) { + registerHuggingFaceInstrumentation(); + registerHuggingFaceTransformersInstrumentation(); + } + + if (integrations.openrouter !== false) { + registerOpenRouterInstrumentation(); + } + + if (integrations.openrouterAgent !== false) { + registerOpenRouterAgentInstrumentation(); + } + + if (integrations.mistral !== false) { + registerMistralInstrumentation(); + } + + if (integrations.ollama !== false) { + registerOllamaInstrumentation(); + } + + if (integrations.googleADK !== false) { + registerGoogleADKInstrumentation(); + } + + if (integrations.cohere !== false) { + registerCohereInstrumentation(); + } + + if (integrations.voyageai !== false) { + registerVoyageAIInstrumentation(); + } + + if (integrations.groq !== false) { + registerGroqInstrumentation(); + } + + if ( + integrations.bedrock !== false && + integrations.awsBedrock !== false && + integrations.awsBedrockRuntime !== false + ) { + registerBedrockRuntimeInstrumentation(); + } + + if (integrations.genkit !== false) { + registerGenkitInstrumentation(); + } + + if (integrations.gitHubCopilot !== false) { + registerGitHubCopilotInstrumentation(); + } + + if (integrations.piCodingAgent !== false) { + registerPiCodingAgentInstrumentation(); + } + + if (integrations.strandsAgentSDK !== false) { + registerStrandsAgentSDKInstrumentation(); + } + + if (integrations.cloudflareAIChat !== false) { + registerCloudflareAIChatInstrumentation(); + } + + if (integrations.cloudflareAgents !== false) { + registerCloudflareAgentsInstrumentation(); + } + + if (integrations.langchain !== false && integrations.langgraph !== false) { + registerLangChainInstrumentation(); + } + + if (integrations.langsmith !== false) { + registerLangSmithInstrumentation({ + skipLangChainRuns: integrations.langchain !== false, + }); + } + + // Mastra is intentionally not wired here: `@mastra/core` ships its own + // ObservabilityExporter contract, and `BraintrustObservabilityExporter` + // (wrappers/mastra.ts) is auto-installed by the loader patch in + // `auto-instrumentations/loader/mastra-observability-patch.ts` rather than + // by an instrumentation consumer / global hook subscription. +} diff --git a/js/src/instrumentation/plugins/cloudflare-agents-plugin.ts b/js/src/instrumentation/plugins/cloudflare-agents-plugin.ts deleted file mode 100644 index d9813f419..000000000 --- a/js/src/instrumentation/plugins/cloudflare-agents-plugin.ts +++ /dev/null @@ -1,140 +0,0 @@ -import { debugLogger } from "../../debug-logger"; -import type { IsoChannelHandlers } from "../../isomorph"; -import { _internalStartSpanWithContext } from "../../logger"; -import type { Span } from "../../logger"; -import { - INSTRUMENTATION_NAMES, - withSpanInstrumentationName, -} from "../../span-origin"; -import { SpanTypeAttribute } from "../../../util/index"; -import { BasePlugin } from "../core"; -import type { ChannelMessage } from "../core/channel-definitions"; -import { cloudflareAgentsChannels } from "./cloudflare-agents-channels"; - -const CLOUDFLARE_WORKERS_CONTEXT = { - span_origin: { - environment: { type: "server", name: "cloudflare_workers" }, - }, -}; - -export class CloudflareAgentsPlugin extends BasePlugin { - protected onEnable(): void { - const channel = cloudflareAgentsChannels.runAgentTool.tracingChannel(); - const spans = new WeakMap(); - const handlers: IsoChannelHandlers< - ChannelMessage - > = { - start: (event) => { - try { - const agentClass = event.arguments[0]; - const options = event.arguments[1]; - if (ownValue(options, "detached")) { - return; - } - - const name = ownValue(agentClass, "name"); - if (typeof name !== "string" || name.length === 0) { - debugLogger.warn( - "Skipping Cloudflare Agents runAgentTool span because the child agent class has no name.", - ); - return; - } - - const span = _internalStartSpanWithContext( - withSpanInstrumentationName( - { - name, - spanAttributes: { type: SpanTypeAttribute.TOOL }, - event: { - input: ownValue(options, "input"), - }, - }, - INSTRUMENTATION_NAMES.CLOUDFLARE_AGENTS, - ), - CLOUDFLARE_WORKERS_CONTEXT, - ); - spans.set(event, span); - } catch (error) { - logInstrumentationError("start", error); - } - }, - asyncEnd: (event) => { - const span = spans.get(event); - if (!span) { - return; - } - spans.delete(event); - - try { - const status = ownValue(event.result, "status"); - if (status === "completed") { - span.log({ output: ownValue(event.result, "output") }); - } else { - const error = ownValue(event.result, "error"); - if (typeof error === "string") { - span.log({ error }); - } - } - } catch (error) { - logInstrumentationError("completion", error); - } finally { - safelyEndSpan(span); - } - }, - error: (event) => { - const span = spans.get(event); - if (!span) { - return; - } - spans.delete(event); - - try { - span.log({ error: event.error }); - } catch (error) { - logInstrumentationError("rejection", error); - } finally { - safelyEndSpan(span); - } - }, - }; - - channel.subscribe(handlers); - this.unsubscribers.push(() => channel.unsubscribe(handlers)); - } - - protected onDisable(): void { - for (const unsubscribe of this.unsubscribers) { - unsubscribe(); - } - this.unsubscribers = []; - } -} - -function ownValue(value: unknown, key: PropertyKey): unknown { - if (!isObjectLike(value)) { - return undefined; - } - const descriptor = Object.getOwnPropertyDescriptor(value, key); - return descriptor && "value" in descriptor ? descriptor.value : undefined; -} - -function isObjectLike(value: unknown): value is object { - return ( - (typeof value === "object" && value !== null) || typeof value === "function" - ); -} - -function safelyEndSpan(span: Span): void { - try { - span.end(); - } catch (error) { - logInstrumentationError("span end", error); - } -} - -function logInstrumentationError(operation: string, error: unknown): void { - debugLogger.error( - `Failed to process Cloudflare Agents ${operation} instrumentation:`, - error, - ); -} diff --git a/js/src/instrumentation/plugins/openai-plugin.ts b/js/src/instrumentation/plugins/openai-plugin.ts deleted file mode 100644 index d1ca89dcc..000000000 --- a/js/src/instrumentation/plugins/openai-plugin.ts +++ /dev/null @@ -1,613 +0,0 @@ -import { BasePlugin } from "../core"; -import { - traceAsyncChannel, - traceStreamingChannel, - traceSyncStreamChannel, - unsubscribeAll, -} from "../core/channel-tracing"; -import { Attachment } from "../../logger"; -import { SpanTypeAttribute, isObject } from "../../../util/index"; -import { getCurrentUnixTimestamp } from "../../util"; -import { processInputAttachments } from "../../wrappers/attachment-utils"; -import { openAIChannels } from "./openai-channels"; -import { - BRAINTRUST_CACHED_STREAM_METRIC, - getCachedMetricFromHeaders, - parseMetricsFromUsage, -} from "../../openai-utils"; -import type { - OpenAIChatChoice, - OpenAIChatCompletionChunk, - OpenAIChatLogprobs, - OpenAIResponseStreamEvent, -} from "../../vendor-sdk-types/openai"; - -/** - * Plugin for OpenAI SDK instrumentation. - * - * Handles instrumentation for: - * - Chat completions (streaming and non-streaming) - * - Embeddings - * - Moderations - * - Beta API (parse, stream) - * - Responses API (create, stream, parse, compact) - */ -export class OpenAIPlugin extends BasePlugin { - constructor() { - super(); - } - - protected onEnable(): void { - // Chat Completions - supports streaming - this.unsubscribers.push( - traceStreamingChannel(openAIChannels.chatCompletionsCreate, { - name: "Chat Completion", - type: SpanTypeAttribute.LLM, - extractInput: ([params]) => { - const { messages, ...metadata } = params; - return { - input: processInputAttachments(messages), - metadata: { ...metadata, provider: "openai" }, - }; - }, - extractOutput: (result) => { - return result?.choices; - }, - extractMetrics: (result, startTime, endEvent) => { - const metrics = withCachedMetric( - parseMetricsFromUsage(result?.usage), - result, - endEvent, - ); - if (startTime) { - metrics.time_to_first_token = getCurrentUnixTimestamp() - startTime; - } - return metrics; - }, - aggregateChunks: aggregateChatCompletionChunks, - }), - ); - - // Embeddings - this.unsubscribers.push( - traceAsyncChannel(openAIChannels.embeddingsCreate, { - name: "Embedding", - type: SpanTypeAttribute.LLM, - extractInput: ([params]) => { - const { input, ...metadata } = params; - return { - input, - metadata: { ...metadata, provider: "openai" }, - }; - }, - extractOutput: (result) => { - const embedding = result?.data?.[0]?.embedding; - return Array.isArray(embedding) - ? { embedding_length: embedding.length } - : undefined; - }, - extractMetrics: (result, _startTime, endEvent) => { - return withCachedMetric( - parseMetricsFromUsage(result?.usage), - result, - endEvent, - ); - }, - }), - ); - - // Beta Chat Completions Parse - this.unsubscribers.push( - traceStreamingChannel(openAIChannels.betaChatCompletionsParse, { - name: "Chat Completion", - type: SpanTypeAttribute.LLM, - extractInput: ([params]) => { - const { messages, ...metadata } = params; - return { - input: processInputAttachments(messages), - metadata: { ...metadata, provider: "openai" }, - }; - }, - extractOutput: (result) => { - return result?.choices; - }, - extractMetrics: (result, startTime, endEvent) => { - const metrics = withCachedMetric( - parseMetricsFromUsage(result?.usage), - result, - endEvent, - ); - if (startTime) { - metrics.time_to_first_token = getCurrentUnixTimestamp() - startTime; - } - return metrics; - }, - aggregateChunks: aggregateChatCompletionChunks, - }), - ); - - // Beta Chat Completions Stream (sync method returning event-based stream) - this.unsubscribers.push( - traceSyncStreamChannel(openAIChannels.betaChatCompletionsStream, { - name: "Chat Completion", - type: SpanTypeAttribute.LLM, - extractInput: ([params]) => { - const { messages, ...metadata } = params; - return { - input: processInputAttachments(messages), - metadata: { ...metadata, provider: "openai" }, - }; - }, - }), - ); - - // Moderations - this.unsubscribers.push( - traceAsyncChannel(openAIChannels.moderationsCreate, { - name: "Moderation", - type: SpanTypeAttribute.LLM, - extractInput: ([params]) => { - const { input, ...metadata } = params; - return { - input, - metadata: { ...metadata, provider: "openai" }, - }; - }, - extractOutput: (result) => { - return result?.results; - }, - extractMetrics: (result, _startTime, endEvent) => { - return withCachedMetric( - parseMetricsFromUsage(result?.usage), - result, - endEvent, - ); - }, - }), - ); - - // Responses API - create (supports streaming via stream=true param) - this.unsubscribers.push( - traceStreamingChannel(openAIChannels.responsesCreate, { - name: "openai.responses.create", - type: SpanTypeAttribute.LLM, - extractInput: ([params]) => { - const { input, ...metadata } = params; - return { - input: processInputAttachments(input), - metadata: { ...metadata, provider: "openai" }, - }; - }, - extractOutput: (result) => { - return processImagesInOutput(result?.output); - }, - extractMetadata: (result) => { - if (!result) { - return undefined; - } - const { output: _output, usage: _usage, ...metadata } = result; - return Object.keys(metadata).length > 0 ? metadata : undefined; - }, - extractMetrics: (result, startTime, endEvent) => { - const metrics = withCachedMetric( - parseMetricsFromUsage(result?.usage), - result, - endEvent, - ); - if (startTime) { - metrics.time_to_first_token = getCurrentUnixTimestamp() - startTime; - } - return metrics; - }, - aggregateChunks: aggregateResponseStreamEvents, - }), - ); - - // Responses API - stream (sync method returning event-based stream) - this.unsubscribers.push( - traceSyncStreamChannel(openAIChannels.responsesStream, { - name: "openai.responses.create", - type: SpanTypeAttribute.LLM, - extractInput: ([params]) => { - const { input, ...metadata } = params; - return { - input: processInputAttachments(input), - metadata: { ...metadata, provider: "openai" }, - }; - }, - extractFromEvent: (event) => { - if (event.type !== "response.completed" || !event.response) { - return {}; - } - - const response = event.response; - const data: Record = {}; - - if (response.output !== undefined) { - data.output = processImagesInOutput(response.output); - } - - const { usage: _usage, output: _output, ...metadata } = response; - if (Object.keys(metadata).length > 0) { - data.metadata = metadata; - } - - data.metrics = parseMetricsFromUsage(response.usage); - return data; - }, - }), - ); - - // Responses API - parse - this.unsubscribers.push( - traceStreamingChannel(openAIChannels.responsesParse, { - name: "openai.responses.parse", - type: SpanTypeAttribute.LLM, - extractInput: ([params]) => { - const { input, ...metadata } = params; - return { - input: processInputAttachments(input), - metadata: { ...metadata, provider: "openai" }, - }; - }, - extractOutput: (result) => { - return processImagesInOutput(result?.output); - }, - extractMetadata: (result) => { - if (!result) { - return undefined; - } - const { output: _output, usage: _usage, ...metadata } = result; - return Object.keys(metadata).length > 0 ? metadata : undefined; - }, - extractMetrics: (result, startTime, endEvent) => { - const metrics = withCachedMetric( - parseMetricsFromUsage(result?.usage), - result, - endEvent, - ); - if (startTime) { - metrics.time_to_first_token = getCurrentUnixTimestamp() - startTime; - } - return metrics; - }, - aggregateChunks: aggregateResponseStreamEvents, - }), - ); - - // Responses API - compact - this.unsubscribers.push( - traceAsyncChannel(openAIChannels.responsesCompact, { - name: "openai.responses.compact", - type: SpanTypeAttribute.LLM, - extractInput: ([params]) => { - const { input, ...metadata } = params; - return { - input: processInputAttachments(input), - metadata: { ...metadata, provider: "openai" }, - }; - }, - extractOutput: (result) => { - return processImagesInOutput(result?.output); - }, - extractMetadata: (result) => { - if (!result) { - return undefined; - } - const { output: _output, usage: _usage, ...metadata } = result; - return Object.keys(metadata).length > 0 ? metadata : undefined; - }, - extractMetrics: (result, startTime, endEvent) => { - const metrics = withCachedMetric( - parseMetricsFromUsage(result?.usage), - result, - endEvent, - ); - if (startTime) { - metrics.time_to_first_token = getCurrentUnixTimestamp() - startTime; - } - return metrics; - }, - }), - ); - } - - protected onDisable(): void { - this.unsubscribers = unsubscribeAll(this.unsubscribers); - } -} - -function getCachedMetricFromEndEvent(endEvent: unknown): number | undefined { - if (!isObject(endEvent)) { - return undefined; - } - - const response = (endEvent as Record).response; - if (!isObject(response)) { - return undefined; - } - - const headers = (response as { headers?: unknown }).headers; - if (!headers || typeof (headers as Headers).get !== "function") { - return undefined; - } - - return getCachedMetricFromHeaders(headers as Headers); -} - -function withCachedMetric( - metrics: Record, - result: unknown, - endEvent?: unknown, -): Record { - if (metrics.cached !== undefined) { - return metrics; - } - - const cachedFromEvent = getCachedMetricFromEndEvent(endEvent); - if (cachedFromEvent !== undefined) { - return { - ...metrics, - cached: cachedFromEvent, - }; - } - - if (!isObject(result)) { - return metrics; - } - - const cached = (result as Record)[ - BRAINTRUST_CACHED_STREAM_METRIC - ]; - - if (typeof cached !== "number") { - return metrics; - } - - return { - ...metrics, - cached, - }; -} - -/** - * Process output to convert base64 images to attachments. - * Used for Responses API image generation output. - */ -export function processImagesInOutput(output: any): any { - if (Array.isArray(output)) { - return output.map(processImagesInOutput); - } - - if (isObject(output)) { - if ( - output.type === "image_generation_call" && - output.result && - typeof output.result === "string" - ) { - const fileExtension = output.output_format || "png"; - const contentType = `image/${fileExtension}`; - - const baseFilename = - output.revised_prompt && typeof output.revised_prompt === "string" - ? output.revised_prompt.slice(0, 50).replace(/[^a-zA-Z0-9]/g, "_") - : "generated_image"; - const filename = `${baseFilename}.${fileExtension}`; - - // Convert base64 string to Blob - const binaryString = atob(output.result); - const bytes = new Uint8Array(binaryString.length); - for (let i = 0; i < binaryString.length; i++) { - bytes[i] = binaryString.charCodeAt(i); - } - const blob = new Blob([bytes], { type: contentType }); - - const attachment = new Attachment({ - data: blob, - filename: filename, - contentType: contentType, - }); - - return { - ...output, - result: attachment, - }; - } - } - - return output; -} - -function mergeLogprobTokens( - existing: OpenAIChatLogprobs["content"] | OpenAIChatLogprobs["refusal"], - incoming: OpenAIChatLogprobs["content"] | OpenAIChatLogprobs["refusal"], -): OpenAIChatLogprobs["content"] | OpenAIChatLogprobs["refusal"] { - if (incoming === undefined) { - return existing; - } - - if (incoming === null) { - return existing ?? null; - } - - if (Array.isArray(existing)) { - return [...existing, ...incoming]; - } - - return [...incoming]; -} - -function aggregateChatLogprobs( - existing: OpenAIChatLogprobs | null | undefined, - incoming: OpenAIChatLogprobs | null | undefined, -): OpenAIChatLogprobs | null | undefined { - if (incoming === undefined) { - return existing; - } - - if (incoming === null) { - return existing ?? null; - } - - const aggregated: OpenAIChatLogprobs = - existing && existing !== null - ? { ...existing, ...incoming } - : { ...incoming }; - - const content = mergeLogprobTokens(existing?.content, incoming.content); - if (content !== undefined) { - aggregated.content = content; - } - - const refusal = mergeLogprobTokens(existing?.refusal, incoming.refusal); - if (refusal !== undefined) { - aggregated.refusal = refusal; - } - - return aggregated; -} - -/** - * Aggregate chat completion chunks into a single response. - * Combines role (first), content (concatenated), tool_calls (by id), - * finish_reason (last), and usage (last chunk). - */ -export function aggregateChatCompletionChunks( - chunks: OpenAIChatCompletionChunk[], - streamResult?: unknown, - endEvent?: unknown, -): { - output: OpenAIChatChoice[]; - metrics: Record; -} { - let role = undefined; - let content = undefined; - let refusal = undefined; - let tool_calls = undefined; - let logprobs: OpenAIChatLogprobs | null | undefined = undefined; - let finish_reason = undefined; - let metrics: Record = {}; - - for (const chunk of chunks) { - if (chunk.usage) { - metrics = { - ...metrics, - ...parseMetricsFromUsage(chunk.usage), - }; - } - - const choice = chunk.choices?.[0]; - if (!choice) { - continue; - } - - if (choice.finish_reason) { - finish_reason = choice.finish_reason; - } - - logprobs = aggregateChatLogprobs(logprobs, choice.logprobs); - - const delta = choice.delta; - if (!delta) { - continue; - } - - if (delta.finish_reason) { - finish_reason = delta.finish_reason; - } - - if (!role && delta.role) { - role = delta.role; - } - - if (delta.content) { - content = (content || "") + delta.content; - } - - if (delta.refusal) { - refusal = (refusal || "") + delta.refusal; - } - - if (delta.tool_calls) { - const toolDelta = delta.tool_calls[0]; - if ( - !tool_calls || - (toolDelta.id && tool_calls[tool_calls.length - 1].id !== toolDelta.id) - ) { - tool_calls = [ - ...(tool_calls || []), - { - id: toolDelta.id, - type: toolDelta.type, - function: toolDelta.function, - }, - ]; - } else { - tool_calls[tool_calls.length - 1].function.arguments += - toolDelta.function.arguments; - } - } - } - - metrics = withCachedMetric(metrics, streamResult, endEvent); - - return { - metrics, - output: [ - { - index: 0, - message: { - role, - content, - ...(refusal !== undefined ? { refusal } : {}), - tool_calls, - }, - logprobs: logprobs ?? null, - finish_reason, - }, - ], - }; -} - -function aggregateResponseStreamEvents( - chunks: OpenAIResponseStreamEvent[], - _streamResult?: unknown, - endEvent?: unknown, -): { - output: any; - metrics: Record; - metadata?: Record; -} { - let output: any = undefined; - let metrics: Record = {}; - let metadata: Record | undefined = undefined; - - for (const chunk of chunks) { - if (!chunk || !chunk.type || !chunk.response) { - continue; - } - if (chunk.type !== "response.completed") { - continue; - } - - const response = chunk.response; - if (response?.output !== undefined) { - output = processImagesInOutput(response.output); - } - - const { usage: _usage, output: _output, ...rest } = response || {}; - if (Object.keys(rest).length > 0) { - metadata = rest; - } - - metrics = parseMetricsFromUsage(response?.usage); - } - - return { - output, - metrics: withCachedMetric(metrics, undefined, endEvent), - ...(metadata !== undefined ? { metadata } : {}), - }; -} - -export { parseMetricsFromUsage }; diff --git a/js/src/instrumentation/plugins/ai-sdk-channels.ts b/js/src/instrumentation/providers/ai-sdk-channels.ts similarity index 100% rename from js/src/instrumentation/plugins/ai-sdk-channels.ts rename to js/src/instrumentation/providers/ai-sdk-channels.ts diff --git a/js/src/instrumentation/plugins/ai-sdk-plugin.streaming.test.ts b/js/src/instrumentation/providers/ai-sdk-instrumentation.streaming.test.ts similarity index 92% rename from js/src/instrumentation/plugins/ai-sdk-plugin.streaming.test.ts rename to js/src/instrumentation/providers/ai-sdk-instrumentation.streaming.test.ts index 8803708c3..1921dfa02 100644 --- a/js/src/instrumentation/plugins/ai-sdk-plugin.streaming.test.ts +++ b/js/src/instrumentation/providers/ai-sdk-instrumentation.streaming.test.ts @@ -22,7 +22,6 @@ import { } from "../../wrappers/ai-sdk/harness-agent-context"; import { workflowAgentWrapperSpanCountForTesting } from "../../wrappers/ai-sdk/workflow-agent-context"; import { aiSDKChannels } from "./ai-sdk-channels"; -import { AISDKPlugin } from "./ai-sdk-plugin"; try { configureNode(); @@ -41,7 +40,7 @@ describe("AI SDK streaming instrumentation", () => { beforeEach(() => { backgroundLogger = _exportsForTestingOnly.useTestBackgroundLogger(); initLogger({ - projectName: "ai-sdk-plugin.streaming.test.ts", + projectName: "ai-sdk-instrumentation.streaming.test.ts", projectId: "test-project-id", }); }); @@ -890,144 +889,130 @@ describe("AI SDK streaming instrumentation", () => { test("baseStream patch preserves derived stream getters", async () => { expect(await backgroundLogger.drain()).toHaveLength(0); - const plugin = new AISDKPlugin(); - plugin.enable(); - - try { - let chunkSent = false; - const result = (await aiSDKChannels.streamText.tracePromise( - async () => { - const resultRecord = { - baseStream: new ReadableStream({ - pull(controller) { - if (chunkSent) { - controller.close(); - return; - } - - chunkSent = true; - controller.enqueue({ - type: "text-delta", - id: "text-1", - delta: "fresh", - }); - }, - }), - text: Promise.resolve("fresh"), - } as any; - - Object.defineProperty(resultRecord, "textStream", { - configurable: true, - enumerable: true, - get() { - const [textBranch, baseBranch] = this.baseStream.tee(); - this.baseStream = baseBranch; - return textBranch.pipeThrough( - new TransformStream({ - transform(chunk: any, controller) { - if (chunk.type === "text-delta") { - controller.enqueue(chunk.delta); - } - }, - }), - ); - }, - }); + let chunkSent = false; + const result = (await aiSDKChannels.streamText.tracePromise( + async () => { + const resultRecord = { + baseStream: new ReadableStream({ + pull(controller) { + if (chunkSent) { + controller.close(); + return; + } - return resultRecord; - }, - { - arguments: [ - { - model: "mock-stream-model", - prompt: "Reply with fresh.", + chunkSent = true; + controller.enqueue({ + type: "text-delta", + id: "text-1", + delta: "fresh", + }); }, - ], - } as any, - )) as any; + }), + text: Promise.resolve("fresh"), + } as any; + + Object.defineProperty(resultRecord, "textStream", { + configurable: true, + enumerable: true, + get() { + const [textBranch, baseBranch] = this.baseStream.tee(); + this.baseStream = baseBranch; + return textBranch.pipeThrough( + new TransformStream({ + transform(chunk: any, controller) { + if (chunk.type === "text-delta") { + controller.enqueue(chunk.delta); + } + }, + }), + ); + }, + }); - expect( - Object.getOwnPropertyDescriptor(result, "textStream")?.get, - ).toEqual(expect.any(Function)); + return resultRecord; + }, + { + arguments: [ + { + model: "mock-stream-model", + prompt: "Reply with fresh.", + }, + ], + } as any, + )) as any; - let firstText = ""; - for await (const chunk of result.textStream) { - firstText += chunk; - } + expect(Object.getOwnPropertyDescriptor(result, "textStream")?.get).toEqual( + expect.any(Function), + ); - let secondText = ""; - for await (const chunk of result.textStream) { - secondText += chunk; - } + let firstText = ""; + for await (const chunk of result.textStream) { + firstText += chunk; + } - expect(firstText).toBe("fresh"); - expect(secondText).toBe("fresh"); - } finally { - plugin.disable(); + let secondText = ""; + for await (const chunk of result.textStream) { + secondText += chunk; } + + expect(firstText).toBe("fresh"); + expect(secondText).toBe("fresh"); }); test("async iterable stream accessors preserve ReadableStream methods", async () => { expect(await backgroundLogger.drain()).toHaveLength(0); - const plugin = new AISDKPlugin(); - plugin.enable(); - - try { - const result = (await aiSDKChannels.streamText.tracePromise( - async () => { - const resultRecord = { - stream: new ReadableStream({ - start(controller) { - controller.enqueue("v7"); - controller.close(); - }, - }), - text: Promise.resolve("v7"), - } as any; - - Object.defineProperty(resultRecord, "textStream", { - configurable: true, - enumerable: true, - get() { - return this.stream.pipeThrough( - new TransformStream({ - transform(chunk: string, controller) { - controller.enqueue(chunk.toUpperCase()); - }, - }), - ); + const result = (await aiSDKChannels.streamText.tracePromise( + async () => { + const resultRecord = { + stream: new ReadableStream({ + start(controller) { + controller.enqueue("v7"); + controller.close(); }, - }); + }), + text: Promise.resolve("v7"), + } as any; + + Object.defineProperty(resultRecord, "textStream", { + configurable: true, + enumerable: true, + get() { + return this.stream.pipeThrough( + new TransformStream({ + transform(chunk: string, controller) { + controller.enqueue(chunk.toUpperCase()); + }, + }), + ); + }, + }); - return resultRecord; - }, - { - arguments: [ - { - model: "mock-v7-stream-model", - prompt: "Reply with v7.", - }, - ], - } as any, - )) as any; + return resultRecord; + }, + { + arguments: [ + { + model: "mock-v7-stream-model", + prompt: "Reply with v7.", + }, + ], + } as any, + )) as any; - expect(result.stream.pipeThrough).toEqual(expect.any(Function)); - expect(result.stream.getReader).toEqual(expect.any(Function)); + expect(result.stream.pipeThrough).toEqual(expect.any(Function)); + expect(result.stream.getReader).toEqual(expect.any(Function)); - const textStream = result.textStream; - expect(textStream.pipeThrough).toEqual(expect.any(Function)); - expect(textStream.getReader).toEqual(expect.any(Function)); + const textStream = result.textStream; + expect(textStream.pipeThrough).toEqual(expect.any(Function)); + expect(textStream.getReader).toEqual(expect.any(Function)); - const reader = textStream.getReader(); - const first = await reader.read(); - const second = await reader.read(); + const reader = textStream.getReader(); + const first = await reader.read(); + const second = await reader.read(); - expect(first).toEqual({ done: false, value: "V7" }); - expect(second).toEqual({ done: true, value: undefined }); - } finally { - plugin.disable(); - } + expect(first).toEqual({ done: false, value: "V7" }); + expect(second).toEqual({ done: true, value: undefined }); }); test("wrapAgentClass instruments all HarnessAgent turn methods without serializing sessions", async () => { diff --git a/js/src/instrumentation/plugins/ai-sdk-plugin.test.ts b/js/src/instrumentation/providers/ai-sdk-instrumentation.test.ts similarity index 96% rename from js/src/instrumentation/plugins/ai-sdk-plugin.test.ts rename to js/src/instrumentation/providers/ai-sdk-instrumentation.test.ts index fcc538deb..60fcbc4bf 100644 --- a/js/src/instrumentation/plugins/ai-sdk-plugin.test.ts +++ b/js/src/instrumentation/providers/ai-sdk-instrumentation.test.ts @@ -1,4 +1,4 @@ -import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; +import { describe, it, expect, vi, beforeEach } from "vitest"; const telemetryMocks = vi.hoisted(() => ({ braintrustAISDKTelemetry: vi.fn(), @@ -23,13 +23,13 @@ vi.mock("../../wrappers/ai-sdk/telemetry", () => ({ })); import { - AISDKPlugin, + registerAISDKInstrumentation, DEFAULT_DENY_OUTPUT_PATHS, processAISDKCallInput, processAISDKWorkflowAgentCallInput, processAISDKWorkflowAgentModelCallInput, processAISDKOutput as processAISDKOutputActual, -} from "./ai-sdk-plugin"; +} from "./ai-sdk-instrumentation"; import iso from "../../isomorph"; import { serializeAISDKToolsForLogging } from "../../wrappers/ai-sdk/tool-serialization"; import { BRAINTRUST_AI_SDK_V7_OPERATION_KEY as AI_SDK_V7_OPERATION_KEY } from "../../vendor-sdk-types/ai-sdk-v7-telemetry"; @@ -39,7 +39,6 @@ type MockTracingChannel = { handlers: any[]; hasSubscribers: boolean; subscribe: ReturnType; - unsubscribe: ReturnType; }; const mockChannels = new Map(); @@ -47,9 +46,7 @@ const mockChannels = new Map(); // Since these are private, we'll test them through the public API // But we'll also add some tests for the exported utility functions -describe("AISDKPlugin", () => { - let plugin: AISDKPlugin; - +describe("registerAISDKInstrumentation", () => { beforeEach(() => { mockChannels.clear(); telemetryMocks.telemetry = { @@ -70,39 +67,10 @@ describe("AISDKPlugin", () => { channel.handlers.push(handlers); channel.hasSubscribers = true; }), - unsubscribe: vi.fn((handlers: any) => { - channel.handlers = channel.handlers.filter( - (candidate) => candidate !== handlers, - ); - channel.hasSubscribers = channel.handlers.length > 0; - return true; - }), }; mockChannels.set(name, channel); return channel; }); - - plugin = new AISDKPlugin(); - }); - - afterEach(() => { - if (plugin) { - plugin.disable(); - } - }); - - describe("constructor", () => { - it("should create plugin with default config", () => { - const defaultPlugin = new AISDKPlugin(); - expect(defaultPlugin).toBeInstanceOf(AISDKPlugin); - }); - - it("should create plugin with custom config", () => { - const customPlugin = new AISDKPlugin({ - denyOutputPaths: ["custom.path"], - }); - expect(customPlugin).toBeInstanceOf(AISDKPlugin); - }); }); describe("WorkflowAgent input extraction", () => { @@ -192,33 +160,9 @@ describe("AISDKPlugin", () => { }); }); - describe("enable/disable", () => { - it("should enable plugin", () => { - expect(() => plugin.enable()).not.toThrow(); - }); - - it("should disable plugin", () => { - plugin.enable(); - expect(() => plugin.disable()).not.toThrow(); - }); - - it("should handle multiple enable calls", () => { - plugin.enable(); - expect(() => plugin.enable()).not.toThrow(); - }); - - it("should handle multiple disable calls", () => { - plugin.enable(); - plugin.disable(); - expect(() => plugin.disable()).not.toThrow(); - }); - - it("should unsubscribe from channels on disable", () => { - plugin.enable(); - plugin.disable(); - // Verify that unsubscribers were called - // This is tested indirectly - if it doesn't throw, unsubscribe worked - expect(true).toBe(true); + describe("registration", () => { + it("registers without throwing", () => { + expect(() => registerAISDKInstrumentation()).not.toThrow(); }); }); @@ -238,7 +182,7 @@ describe("AISDKPlugin", () => { onStepEnd: existingOnStepEnd, }; - plugin.enable(); + registerAISDKInstrumentation(); const channel = mockChannels.get( "orchestrion:ai:createTelemetryDispatcher", @@ -309,7 +253,7 @@ describe("AISDKPlugin", () => { onStart: existingOnStart, }; - plugin.enable(); + registerAISDKInstrumentation(); const channel = mockChannels.get( "orchestrion:ai:createTelemetryDispatcher", @@ -368,7 +312,7 @@ describe("AISDKPlugin", () => { onStart: vi.fn(), }; - plugin.enable(); + registerAISDKInstrumentation(); const channel = mockChannels.get( "orchestrion:ai:createTelemetryDispatcher", @@ -432,7 +376,7 @@ describe("AISDKPlugin", () => { onStart: existingOnStart, }; - plugin.enable(); + registerAISDKInstrumentation(); const channel = mockChannels.get( "orchestrion:ai:createTelemetryDispatcher", @@ -460,7 +404,7 @@ describe("AISDKPlugin", () => { onStart: vi.fn(), }; - plugin.enable(); + registerAISDKInstrumentation(); const channel = mockChannels.get( "orchestrion:ai:createTelemetryDispatcher", diff --git a/js/src/instrumentation/plugins/ai-sdk-plugin.ts b/js/src/instrumentation/providers/ai-sdk-instrumentation.ts similarity index 85% rename from js/src/instrumentation/plugins/ai-sdk-plugin.ts rename to js/src/instrumentation/providers/ai-sdk-instrumentation.ts index e6cbf92f4..8b797e987 100644 --- a/js/src/instrumentation/plugins/ai-sdk-plugin.ts +++ b/js/src/instrumentation/providers/ai-sdk-instrumentation.ts @@ -1,10 +1,9 @@ -import { BasePlugin, toLoggedError } from "../core"; +import { toLoggedError } from "../core"; import { debugLogger } from "../../debug-logger"; import { traceAsyncChannel, traceStreamingChannel, traceSyncStreamChannel, - unsubscribeAll, } from "../core/channel-tracing"; import type { ChannelMessage } from "../core/channel-definitions"; import { isAsyncIterable, patchStreamIfNeeded } from "../core/stream-patcher"; @@ -75,7 +74,7 @@ import type { } from "../../vendor-sdk-types/ai-sdk-v7-telemetry"; import { BRAINTRUST_AI_SDK_V7_OPERATION_KEY as AI_SDK_V7_OPERATION_KEY } from "../../vendor-sdk-types/ai-sdk-v7-telemetry"; -interface AISDKPluginConfig { +interface AISDKInstrumentationConfig { /** * List of JSON paths to remove from output field. * Uses dot notation with array wildcards: "roundtrips[].request.body" @@ -150,10 +149,10 @@ const AI_SDK_V7_TELEMETRY_CALLBACKS = [ ] as const; /** - * AI SDK plugin that subscribes to instrumentation channels + * Internal AI SDK consumer that subscribes to instrumentation channels * and creates Braintrust spans. * - * This plugin handles: + * This consumer handles: * - generateText (async function) * - streamText (function returning stream) * - generateObject (async function) @@ -167,500 +166,455 @@ const AI_SDK_V7_TELEMETRY_CALLBACKS = [ * - ToolLoopAgent.stream (async method returning stream) * - WorkflowAgent.stream (async method returning stream) * - * The plugin automatically extracts: + * The consumer automatically extracts: * - Model and provider information * - Token usage metrics * - Tool calls and structured outputs * - Streaming responses with time-to-first-token */ -export class AISDKPlugin extends BasePlugin { - private config: AISDKPluginConfig; +class AISDKInstrumentationConsumer { + private config: AISDKInstrumentationConfig; - constructor(config: AISDKPluginConfig = {}) { - super(); + constructor(config: AISDKInstrumentationConfig = {}) { this.config = config; } - protected onEnable(): void { + public register(): void { this.subscribeToAISDK(); } - protected onDisable(): void { - this.unsubscribers = unsubscribeAll(this.unsubscribers); - } - private subscribeToAISDK(): void { const denyOutputPaths = this.config.denyOutputPaths || DEFAULT_DENY_OUTPUT_PATHS; - this.unsubscribers.push(subscribeToAISDKV7TelemetryDispatcher()); - this.unsubscribers.push(subscribeToHarnessAgentCreateSession()); - this.unsubscribers.push( - subscribeToHarnessContinuation( - harnessAgentChannels.continueGenerate, - denyOutputPaths, - ), - subscribeToHarnessContinuation( - harnessAgentChannels.continueStream, - denyOutputPaths, - ), + subscribeToAISDKV7TelemetryDispatcher(); + subscribeToHarnessAgentCreateSession(); + subscribeToHarnessContinuation( + harnessAgentChannels.continueGenerate, + denyOutputPaths, + ); + subscribeToHarnessContinuation( + harnessAgentChannels.continueStream, + denyOutputPaths, ); // generateText - async function that may return streams - this.unsubscribers.push( - traceStreamingChannel(aiSDKChannels.generateText, { - name: "generateText", - type: SpanTypeAttribute.FUNCTION, - extractInput: ([params], event, span) => - prepareAISDKCallInput(params, event, span, denyOutputPaths), - extractOutput: (result, endEvent) => { - finalizeAISDKChildTracing(endEvent as { [key: string]: unknown }); - return processAISDKOutput( - result, - resolveDenyOutputPaths(endEvent, denyOutputPaths), - ); - }, - extractMetrics: (result, _startTime, endEvent) => - extractTopLevelAISDKMetrics(result, endEvent), - aggregateChunks: aggregateAISDKChunks, - }), - ); + traceStreamingChannel(aiSDKChannels.generateText, { + name: "generateText", + type: SpanTypeAttribute.FUNCTION, + extractInput: ([params], event, span) => + prepareAISDKCallInput(params, event, span, denyOutputPaths), + extractOutput: (result, endEvent) => { + finalizeAISDKChildTracing(endEvent as { [key: string]: unknown }); + return processAISDKOutput( + result, + resolveDenyOutputPaths(endEvent, denyOutputPaths), + ); + }, + extractMetrics: (result, _startTime, endEvent) => + extractTopLevelAISDKMetrics(result, endEvent), + aggregateChunks: aggregateAISDKChunks, + }); // streamText - function returning stream - this.unsubscribers.push( - traceStreamingChannel(aiSDKChannels.streamText, { - name: "streamText", - type: SpanTypeAttribute.FUNCTION, - shouldTrace: () => currentCloudflareThinkSpan() === undefined, - extractInput: ([params], event, span) => - prepareAISDKCallInput(params, event, span, denyOutputPaths), - extractOutput: (result, endEvent) => - processAISDKOutput( - result, - resolveDenyOutputPaths(endEvent, denyOutputPaths), - ), - extractMetrics: (result, startTime, endEvent) => - extractTopLevelAISDKMetrics(result, endEvent, startTime), - aggregateChunks: aggregateAISDKChunks, - patchResult: ({ endEvent, result, span, startTime }) => - patchAISDKStreamingResult({ - defaultDenyOutputPaths: denyOutputPaths, - endEvent, - result, - span, - startTime, - }), - }), - ); + traceStreamingChannel(aiSDKChannels.streamText, { + name: "streamText", + type: SpanTypeAttribute.FUNCTION, + shouldTrace: () => currentCloudflareThinkSpan() === undefined, + extractInput: ([params], event, span) => + prepareAISDKCallInput(params, event, span, denyOutputPaths), + extractOutput: (result, endEvent) => + processAISDKOutput( + result, + resolveDenyOutputPaths(endEvent, denyOutputPaths), + ), + extractMetrics: (result, startTime, endEvent) => + extractTopLevelAISDKMetrics(result, endEvent, startTime), + aggregateChunks: aggregateAISDKChunks, + patchResult: ({ endEvent, result, span, startTime }) => + patchAISDKStreamingResult({ + defaultDenyOutputPaths: denyOutputPaths, + endEvent, + result, + span, + startTime, + }), + }); // streamText - sync function returning stream (v4+, used by auto-hook) - this.unsubscribers.push( - traceSyncStreamChannel(aiSDKChannels.streamTextSync, { - name: "streamText", - type: SpanTypeAttribute.FUNCTION, - shouldTrace: () => currentCloudflareThinkSpan() === undefined, - extractInput: ([params], event, span) => - prepareAISDKCallInput(params, event, span, denyOutputPaths), - patchResult: ({ endEvent, result, span, startTime }) => - patchAISDKStreamingResult({ - defaultDenyOutputPaths: denyOutputPaths, - endEvent, - result, - span, - startTime, - }), - }), - ); + traceSyncStreamChannel(aiSDKChannels.streamTextSync, { + name: "streamText", + type: SpanTypeAttribute.FUNCTION, + shouldTrace: () => currentCloudflareThinkSpan() === undefined, + extractInput: ([params], event, span) => + prepareAISDKCallInput(params, event, span, denyOutputPaths), + patchResult: ({ endEvent, result, span, startTime }) => + patchAISDKStreamingResult({ + defaultDenyOutputPaths: denyOutputPaths, + endEvent, + result, + span, + startTime, + }), + }); // generateObject - async function that may return streams - this.unsubscribers.push( - traceStreamingChannel(aiSDKChannels.generateObject, { - name: "generateObject", - type: SpanTypeAttribute.FUNCTION, - extractInput: ([params], event, span) => - prepareAISDKCallInput(params, event, span, denyOutputPaths), - extractOutput: (result, endEvent) => { - finalizeAISDKChildTracing(endEvent as { [key: string]: unknown }); - return processAISDKOutput( - result, - resolveDenyOutputPaths(endEvent, denyOutputPaths), - ); - }, - extractMetrics: (result, _startTime, endEvent) => - extractTopLevelAISDKMetrics(result, endEvent), - aggregateChunks: aggregateAISDKChunks, - }), - ); + traceStreamingChannel(aiSDKChannels.generateObject, { + name: "generateObject", + type: SpanTypeAttribute.FUNCTION, + extractInput: ([params], event, span) => + prepareAISDKCallInput(params, event, span, denyOutputPaths), + extractOutput: (result, endEvent) => { + finalizeAISDKChildTracing(endEvent as { [key: string]: unknown }); + return processAISDKOutput( + result, + resolveDenyOutputPaths(endEvent, denyOutputPaths), + ); + }, + extractMetrics: (result, _startTime, endEvent) => + extractTopLevelAISDKMetrics(result, endEvent), + aggregateChunks: aggregateAISDKChunks, + }); // streamObject - function returning stream - this.unsubscribers.push( - traceStreamingChannel(aiSDKChannels.streamObject, { - name: "streamObject", - type: SpanTypeAttribute.FUNCTION, - extractInput: ([params], event, span) => - prepareAISDKCallInput(params, event, span, denyOutputPaths), - extractOutput: (result, endEvent) => - processAISDKOutput( - result, - resolveDenyOutputPaths(endEvent, denyOutputPaths), - ), - extractMetrics: (result, startTime, endEvent) => - extractTopLevelAISDKMetrics(result, endEvent, startTime), - aggregateChunks: aggregateAISDKChunks, - patchResult: ({ endEvent, result, span, startTime }) => - patchAISDKStreamingResult({ - defaultDenyOutputPaths: denyOutputPaths, - endEvent, - result, - span, - startTime, - }), - }), - ); + traceStreamingChannel(aiSDKChannels.streamObject, { + name: "streamObject", + type: SpanTypeAttribute.FUNCTION, + extractInput: ([params], event, span) => + prepareAISDKCallInput(params, event, span, denyOutputPaths), + extractOutput: (result, endEvent) => + processAISDKOutput( + result, + resolveDenyOutputPaths(endEvent, denyOutputPaths), + ), + extractMetrics: (result, startTime, endEvent) => + extractTopLevelAISDKMetrics(result, endEvent, startTime), + aggregateChunks: aggregateAISDKChunks, + patchResult: ({ endEvent, result, span, startTime }) => + patchAISDKStreamingResult({ + defaultDenyOutputPaths: denyOutputPaths, + endEvent, + result, + span, + startTime, + }), + }); // streamObject - sync function returning stream (v4+, used by auto-hook) - this.unsubscribers.push( - traceSyncStreamChannel(aiSDKChannels.streamObjectSync, { - name: "streamObject", - type: SpanTypeAttribute.FUNCTION, - extractInput: ([params], event, span) => - prepareAISDKCallInput(params, event, span, denyOutputPaths), - patchResult: ({ endEvent, result, span, startTime }) => - patchAISDKStreamingResult({ - defaultDenyOutputPaths: denyOutputPaths, - endEvent, - result, - span, - startTime, - }), - }), - ); + traceSyncStreamChannel(aiSDKChannels.streamObjectSync, { + name: "streamObject", + type: SpanTypeAttribute.FUNCTION, + extractInput: ([params], event, span) => + prepareAISDKCallInput(params, event, span, denyOutputPaths), + patchResult: ({ endEvent, result, span, startTime }) => + patchAISDKStreamingResult({ + defaultDenyOutputPaths: denyOutputPaths, + endEvent, + result, + span, + startTime, + }), + }); // embed - async embedding function - this.unsubscribers.push( - traceAsyncChannel(aiSDKChannels.embed, { - name: "embed", - type: SpanTypeAttribute.FUNCTION, - extractInput: ([params], event) => - prepareAISDKEmbedInput(params, event.self), - extractOutput: (result, endEvent) => - processAISDKEmbeddingOutput( - result, - resolveDenyOutputPaths(endEvent, denyOutputPaths), - ), - extractMetrics: (result, _startTime, endEvent) => - extractTopLevelAISDKMetrics(result, endEvent), - }), - ); + traceAsyncChannel(aiSDKChannels.embed, { + name: "embed", + type: SpanTypeAttribute.FUNCTION, + extractInput: ([params], event) => + prepareAISDKEmbedInput(params, event.self), + extractOutput: (result, endEvent) => + processAISDKEmbeddingOutput( + result, + resolveDenyOutputPaths(endEvent, denyOutputPaths), + ), + extractMetrics: (result, _startTime, endEvent) => + extractTopLevelAISDKMetrics(result, endEvent), + }); // embedMany - async embedding batch function - this.unsubscribers.push( - traceAsyncChannel(aiSDKChannels.embedMany, { - name: "embedMany", - type: SpanTypeAttribute.FUNCTION, - extractInput: ([params], event) => - prepareAISDKEmbedInput(params, event.self), - extractOutput: (result, endEvent) => - processAISDKEmbeddingOutput( - result, - resolveDenyOutputPaths(endEvent, denyOutputPaths), - ), - extractMetrics: (result, _startTime, endEvent) => - extractTopLevelAISDKMetrics(result, endEvent), - }), - ); + traceAsyncChannel(aiSDKChannels.embedMany, { + name: "embedMany", + type: SpanTypeAttribute.FUNCTION, + extractInput: ([params], event) => + prepareAISDKEmbedInput(params, event.self), + extractOutput: (result, endEvent) => + processAISDKEmbeddingOutput( + result, + resolveDenyOutputPaths(endEvent, denyOutputPaths), + ), + extractMetrics: (result, _startTime, endEvent) => + extractTopLevelAISDKMetrics(result, endEvent), + }); // rerank - async reranking function - this.unsubscribers.push( - traceAsyncChannel(aiSDKChannels.rerank, { - name: "rerank", - type: SpanTypeAttribute.FUNCTION, - extractInput: ([params], event) => - prepareAISDKRerankInput(params, event.self), - extractOutput: (result, endEvent) => - processAISDKRerankOutput( - result, - resolveDenyOutputPaths(endEvent, denyOutputPaths), - ), - extractMetrics: (result, _startTime, endEvent) => - extractTopLevelAISDKMetrics(result, endEvent), - }), - ); + traceAsyncChannel(aiSDKChannels.rerank, { + name: "rerank", + type: SpanTypeAttribute.FUNCTION, + extractInput: ([params], event) => + prepareAISDKRerankInput(params, event.self), + extractOutput: (result, endEvent) => + processAISDKRerankOutput( + result, + resolveDenyOutputPaths(endEvent, denyOutputPaths), + ), + extractMetrics: (result, _startTime, endEvent) => + extractTopLevelAISDKMetrics(result, endEvent), + }); // Agent.generate - async method - this.unsubscribers.push( - traceStreamingChannel(aiSDKChannels.agentGenerate, { - name: "Agent.generate", - type: SpanTypeAttribute.FUNCTION, - extractInput: ([params], event, span) => - prepareAISDKCallInput(params, event, span, denyOutputPaths, { - agentOwner: true, - }), - extractOutput: (result, endEvent) => { - finalizeAISDKChildTracing(endEvent as { [key: string]: unknown }); - return processAISDKOutput( - result, - resolveDenyOutputPaths(endEvent, denyOutputPaths), - ); - }, - extractMetrics: (result, _startTime, endEvent) => - extractTopLevelAISDKMetrics(result, endEvent), - aggregateChunks: aggregateAISDKChunks, - }), - ); + traceStreamingChannel(aiSDKChannels.agentGenerate, { + name: "Agent.generate", + type: SpanTypeAttribute.FUNCTION, + extractInput: ([params], event, span) => + prepareAISDKCallInput(params, event, span, denyOutputPaths, { + agentOwner: true, + }), + extractOutput: (result, endEvent) => { + finalizeAISDKChildTracing(endEvent as { [key: string]: unknown }); + return processAISDKOutput( + result, + resolveDenyOutputPaths(endEvent, denyOutputPaths), + ); + }, + extractMetrics: (result, _startTime, endEvent) => + extractTopLevelAISDKMetrics(result, endEvent), + aggregateChunks: aggregateAISDKChunks, + }); // Agent.stream - async method returning stream (v5, used by wrapAISDK) - this.unsubscribers.push( - traceStreamingChannel(aiSDKChannels.agentStream, { - name: "Agent.stream", - type: SpanTypeAttribute.FUNCTION, - extractInput: ([params], event, span) => - prepareAISDKCallInput(params, event, span, denyOutputPaths, { - agentOwner: true, - }), - extractOutput: (result, endEvent) => - processAISDKOutput( - result, - resolveDenyOutputPaths(endEvent, denyOutputPaths), - ), - extractMetrics: (result, startTime, endEvent) => - extractTopLevelAISDKMetrics(result, endEvent, startTime), - aggregateChunks: aggregateAISDKChunks, - patchResult: ({ endEvent, result, span, startTime }) => - patchAISDKStreamingResult({ - defaultDenyOutputPaths: denyOutputPaths, - endEvent, - result, - span, - startTime, - }), - }), - ); + traceStreamingChannel(aiSDKChannels.agentStream, { + name: "Agent.stream", + type: SpanTypeAttribute.FUNCTION, + extractInput: ([params], event, span) => + prepareAISDKCallInput(params, event, span, denyOutputPaths, { + agentOwner: true, + }), + extractOutput: (result, endEvent) => + processAISDKOutput( + result, + resolveDenyOutputPaths(endEvent, denyOutputPaths), + ), + extractMetrics: (result, startTime, endEvent) => + extractTopLevelAISDKMetrics(result, endEvent, startTime), + aggregateChunks: aggregateAISDKChunks, + patchResult: ({ endEvent, result, span, startTime }) => + patchAISDKStreamingResult({ + defaultDenyOutputPaths: denyOutputPaths, + endEvent, + result, + span, + startTime, + }), + }); // Agent.stream - sync method returning stream (v5, used by auto-hook) - this.unsubscribers.push( - traceSyncStreamChannel(aiSDKChannels.agentStreamSync, { - name: "Agent.stream", - type: SpanTypeAttribute.FUNCTION, - extractInput: ([params], event, span) => - prepareAISDKCallInput(params, event, span, denyOutputPaths, { - agentOwner: true, - }), - patchResult: ({ endEvent, result, span, startTime }) => - patchAISDKStreamingResult({ - defaultDenyOutputPaths: denyOutputPaths, - endEvent, - result, - span, - startTime, - }), - }), - ); + traceSyncStreamChannel(aiSDKChannels.agentStreamSync, { + name: "Agent.stream", + type: SpanTypeAttribute.FUNCTION, + extractInput: ([params], event, span) => + prepareAISDKCallInput(params, event, span, denyOutputPaths, { + agentOwner: true, + }), + patchResult: ({ endEvent, result, span, startTime }) => + patchAISDKStreamingResult({ + defaultDenyOutputPaths: denyOutputPaths, + endEvent, + result, + span, + startTime, + }), + }); // HarnessAgent.generate - one task span per agent turn - this.unsubscribers.push( - traceStreamingChannel(harnessAgentChannels.generate, { - name: "HarnessAgent.generate", - startSpan: _internalStartSpanWithInitialMerge, - type: SpanTypeAttribute.TASK, - extractInput: ([params], event, span) => - prepareAISDKHarnessAgentInput(params, event.self, span), - extractOutput: (result, endEvent) => - processAISDKOutput( - result, - resolveDenyOutputPaths(endEvent, denyOutputPaths), - ), - extractMetrics: (result) => extractTokenMetrics(result), - aggregateChunks: aggregateAISDKChunks, - }), - ); + traceStreamingChannel(harnessAgentChannels.generate, { + name: "HarnessAgent.generate", + startSpan: _internalStartSpanWithInitialMerge, + type: SpanTypeAttribute.TASK, + extractInput: ([params], event, span) => + prepareAISDKHarnessAgentInput(params, event.self, span), + extractOutput: (result, endEvent) => + processAISDKOutput( + result, + resolveDenyOutputPaths(endEvent, denyOutputPaths), + ), + extractMetrics: (result) => extractTokenMetrics(result), + aggregateChunks: aggregateAISDKChunks, + }); // HarnessAgent.stream - async method returning an AI SDK stream result - this.unsubscribers.push( - traceStreamingChannel(harnessAgentChannels.stream, { - name: "HarnessAgent.stream", - startSpan: _internalStartSpanWithInitialMerge, - type: SpanTypeAttribute.TASK, - extractInput: ([params], event, span) => - prepareAISDKHarnessAgentInput(params, event.self, span), - extractOutput: (result, endEvent) => - processAISDKOutput( - result, - resolveDenyOutputPaths(endEvent, denyOutputPaths), - ), - extractMetrics: (result, startTime) => ({ - ...extractTokenMetrics(result), - ...(startTime === undefined - ? {} - : { - time_to_first_token: getCurrentUnixTimestamp() - startTime, - }), - }), - aggregateChunks: aggregateAISDKChunks, - patchResult: ({ endEvent, result, span, startTime }) => - patchAISDKStreamingResult({ - defaultDenyOutputPaths: denyOutputPaths, - endEvent, - result, - resolvePromiseUsage: true, - span, - startTime, - }), + traceStreamingChannel(harnessAgentChannels.stream, { + name: "HarnessAgent.stream", + startSpan: _internalStartSpanWithInitialMerge, + type: SpanTypeAttribute.TASK, + extractInput: ([params], event, span) => + prepareAISDKHarnessAgentInput(params, event.self, span), + extractOutput: (result, endEvent) => + processAISDKOutput( + result, + resolveDenyOutputPaths(endEvent, denyOutputPaths), + ), + extractMetrics: (result, startTime) => ({ + ...extractTokenMetrics(result), + ...(startTime === undefined + ? {} + : { + time_to_first_token: getCurrentUnixTimestamp() - startTime, + }), }), - ); + aggregateChunks: aggregateAISDKChunks, + patchResult: ({ endEvent, result, span, startTime }) => + patchAISDKStreamingResult({ + defaultDenyOutputPaths: denyOutputPaths, + endEvent, + result, + resolvePromiseUsage: true, + span, + startTime, + }), + }); // Trace a continuation as its own task only when its original turn cannot // be recovered. Known continuations extend the original Harness task. - this.unsubscribers.push( - traceStreamingChannel(harnessAgentChannels.continueGenerate, { - name: "HarnessAgent.continueGenerate", - shouldTrace: (args) => - !harnessContinuationParent(harnessSessionFromArguments(args)), - startSpan: _internalStartSpanWithInitialMerge, - type: SpanTypeAttribute.TASK, - extractInput: ([params], event, span) => - prepareAISDKHarnessAgentInput(params, event.self, span), - extractOutput: (result, endEvent) => - processAISDKOutput( - result, - resolveDenyOutputPaths(endEvent, denyOutputPaths), - ), - extractMetrics: (result) => extractTokenMetrics(result), - aggregateChunks: aggregateAISDKChunks, - }), - ); + traceStreamingChannel(harnessAgentChannels.continueGenerate, { + name: "HarnessAgent.continueGenerate", + shouldTrace: (args) => + !harnessContinuationParent(harnessSessionFromArguments(args)), + startSpan: _internalStartSpanWithInitialMerge, + type: SpanTypeAttribute.TASK, + extractInput: ([params], event, span) => + prepareAISDKHarnessAgentInput(params, event.self, span), + extractOutput: (result, endEvent) => + processAISDKOutput( + result, + resolveDenyOutputPaths(endEvent, denyOutputPaths), + ), + extractMetrics: (result) => extractTokenMetrics(result), + aggregateChunks: aggregateAISDKChunks, + }); - this.unsubscribers.push( - traceStreamingChannel(harnessAgentChannels.continueStream, { - name: "HarnessAgent.continueStream", - shouldTrace: (args) => - !harnessContinuationParent(harnessSessionFromArguments(args)), - startSpan: _internalStartSpanWithInitialMerge, - type: SpanTypeAttribute.TASK, - extractInput: ([params], event, span) => - prepareAISDKHarnessAgentInput(params, event.self, span), - extractOutput: (result, endEvent) => - processAISDKOutput( - result, - resolveDenyOutputPaths(endEvent, denyOutputPaths), - ), - extractMetrics: (result, startTime) => ({ - ...extractTokenMetrics(result), - ...(startTime === undefined - ? {} - : { - time_to_first_token: getCurrentUnixTimestamp() - startTime, - }), - }), - aggregateChunks: aggregateAISDKChunks, - patchResult: ({ endEvent, result, span, startTime }) => - patchAISDKStreamingResult({ - defaultDenyOutputPaths: denyOutputPaths, - endEvent, - result, - resolvePromiseUsage: true, - span, - startTime, - }), + traceStreamingChannel(harnessAgentChannels.continueStream, { + name: "HarnessAgent.continueStream", + shouldTrace: (args) => + !harnessContinuationParent(harnessSessionFromArguments(args)), + startSpan: _internalStartSpanWithInitialMerge, + type: SpanTypeAttribute.TASK, + extractInput: ([params], event, span) => + prepareAISDKHarnessAgentInput(params, event.self, span), + extractOutput: (result, endEvent) => + processAISDKOutput( + result, + resolveDenyOutputPaths(endEvent, denyOutputPaths), + ), + extractMetrics: (result, startTime) => ({ + ...extractTokenMetrics(result), + ...(startTime === undefined + ? {} + : { + time_to_first_token: getCurrentUnixTimestamp() - startTime, + }), }), - ); + aggregateChunks: aggregateAISDKChunks, + patchResult: ({ endEvent, result, span, startTime }) => + patchAISDKStreamingResult({ + defaultDenyOutputPaths: denyOutputPaths, + endEvent, + result, + resolvePromiseUsage: true, + span, + startTime, + }), + }); // ToolLoopAgent.generate - async method - this.unsubscribers.push( - traceStreamingChannel(aiSDKChannels.toolLoopAgentGenerate, { - name: "ToolLoopAgent.generate", - type: SpanTypeAttribute.FUNCTION, - extractInput: ([params], event, span) => - prepareAISDKCallInput(params, event, span, denyOutputPaths, { - agentOwner: true, - }), - extractOutput: (result, endEvent) => { - finalizeAISDKChildTracing(endEvent as { [key: string]: unknown }); - return processAISDKOutput( - result, - resolveDenyOutputPaths(endEvent, denyOutputPaths), - ); - }, - extractMetrics: (result, _startTime, endEvent) => - extractTopLevelAISDKMetrics(result, endEvent), - aggregateChunks: aggregateAISDKChunks, - }), - ); + traceStreamingChannel(aiSDKChannels.toolLoopAgentGenerate, { + name: "ToolLoopAgent.generate", + type: SpanTypeAttribute.FUNCTION, + extractInput: ([params], event, span) => + prepareAISDKCallInput(params, event, span, denyOutputPaths, { + agentOwner: true, + }), + extractOutput: (result, endEvent) => { + finalizeAISDKChildTracing(endEvent as { [key: string]: unknown }); + return processAISDKOutput( + result, + resolveDenyOutputPaths(endEvent, denyOutputPaths), + ); + }, + extractMetrics: (result, _startTime, endEvent) => + extractTopLevelAISDKMetrics(result, endEvent), + aggregateChunks: aggregateAISDKChunks, + }); // ToolLoopAgent.stream - async method returning stream - this.unsubscribers.push( - traceStreamingChannel(aiSDKChannels.toolLoopAgentStream, { - name: "ToolLoopAgent.stream", - type: SpanTypeAttribute.FUNCTION, - extractInput: ([params], event, span) => - prepareAISDKCallInput(params, event, span, denyOutputPaths, { - agentOwner: true, - }), - extractOutput: (result, endEvent) => - processAISDKOutput( - result, - resolveDenyOutputPaths(endEvent, denyOutputPaths), - ), - extractMetrics: (result, startTime, endEvent) => - extractTopLevelAISDKMetrics(result, endEvent, startTime), - aggregateChunks: aggregateAISDKChunks, - patchResult: ({ endEvent, result, span, startTime }) => - patchAISDKStreamingResult({ - defaultDenyOutputPaths: denyOutputPaths, - endEvent, - result, - span, - startTime, - }), - }), - ); + traceStreamingChannel(aiSDKChannels.toolLoopAgentStream, { + name: "ToolLoopAgent.stream", + type: SpanTypeAttribute.FUNCTION, + extractInput: ([params], event, span) => + prepareAISDKCallInput(params, event, span, denyOutputPaths, { + agentOwner: true, + }), + extractOutput: (result, endEvent) => + processAISDKOutput( + result, + resolveDenyOutputPaths(endEvent, denyOutputPaths), + ), + extractMetrics: (result, startTime, endEvent) => + extractTopLevelAISDKMetrics(result, endEvent, startTime), + aggregateChunks: aggregateAISDKChunks, + patchResult: ({ endEvent, result, span, startTime }) => + patchAISDKStreamingResult({ + defaultDenyOutputPaths: denyOutputPaths, + endEvent, + result, + span, + startTime, + }), + }); // WorkflowAgent.stream - async method returning stream - this.unsubscribers.push( - traceStreamingChannel(aiSDKChannels.workflowAgentStream, { - name: "WorkflowAgent.stream", - type: SpanTypeAttribute.FUNCTION, - extractInput: ([params], event, span) => - prepareAISDKWorkflowAgentStreamInput( - params, - event, - span, - denyOutputPaths, - ), - extractOutput: (result, endEvent) => { - finalizeAISDKChildTracing(endEvent as { [key: string]: unknown }); - return processAISDKOutput( - result, - resolveDenyOutputPaths(endEvent, denyOutputPaths), - ); - }, - extractMetrics: (result, _startTime, endEvent) => - extractTopLevelAISDKMetrics(result, endEvent), - aggregateChunks: aggregateAISDKChunks, - onComplete: ({ span }) => { - unregisterWorkflowAgentWrapperSpan(span); - }, - onError: ({ event, span }) => { - finalizeAISDKChildTracing(event as { [key: string]: unknown }); - unregisterWorkflowAgentWrapperSpan(span); - }, - patchResult: ({ endEvent, result, span, startTime }) => - patchAISDKStreamingResult({ - defaultDenyOutputPaths: denyOutputPaths, - endEvent, - onComplete: () => unregisterWorkflowAgentWrapperSpan(span), - onCancel: () => unregisterWorkflowAgentWrapperSpan(span), - onError: () => unregisterWorkflowAgentWrapperSpan(span), - result, - span, - startTime, - }), - }), - ); + traceStreamingChannel(aiSDKChannels.workflowAgentStream, { + name: "WorkflowAgent.stream", + type: SpanTypeAttribute.FUNCTION, + extractInput: ([params], event, span) => + prepareAISDKWorkflowAgentStreamInput( + params, + event, + span, + denyOutputPaths, + ), + extractOutput: (result, endEvent) => { + finalizeAISDKChildTracing(endEvent as { [key: string]: unknown }); + return processAISDKOutput( + result, + resolveDenyOutputPaths(endEvent, denyOutputPaths), + ); + }, + extractMetrics: (result, _startTime, endEvent) => + extractTopLevelAISDKMetrics(result, endEvent), + aggregateChunks: aggregateAISDKChunks, + onComplete: ({ span }) => { + unregisterWorkflowAgentWrapperSpan(span); + }, + onError: ({ event, span }) => { + finalizeAISDKChildTracing(event as { [key: string]: unknown }); + unregisterWorkflowAgentWrapperSpan(span); + }, + patchResult: ({ endEvent, result, span, startTime }) => + patchAISDKStreamingResult({ + defaultDenyOutputPaths: denyOutputPaths, + endEvent, + onComplete: () => unregisterWorkflowAgentWrapperSpan(span), + onCancel: () => unregisterWorkflowAgentWrapperSpan(span), + onError: () => unregisterWorkflowAgentWrapperSpan(span), + result, + span, + startTime, + }), + }); } } -function subscribeToHarnessAgentCreateSession(): () => void { +function subscribeToHarnessAgentCreateSession(): void { const channel = harnessAgentChannels.createSession.tracingChannel(); const parents = new WeakMap(); const handlers: IsoChannelHandlers< @@ -682,7 +636,6 @@ function subscribeToHarnessAgentCreateSession(): () => void { }; channel.subscribe(handlers); - return () => channel.unsubscribe(handlers); } type HarnessContinuationChannel = typeof harnessAgentChannels.continueGenerate; @@ -700,14 +653,11 @@ function harnessContinuationParentFromEvent( function subscribeToHarnessContinuation( continuationChannel: HarnessContinuationChannel, defaultDenyOutputPaths: string[], -): () => void { +): void { const channel = continuationChannel.tracingChannel(); const parents = new WeakMap(); const startTimes = new WeakMap(); - const unbindParentStore = bindHarnessTurnParentToStart( - channel, - harnessContinuationParentFromEvent, - ); + bindHarnessTurnParentToStart(channel, harnessContinuationParentFromEvent); const handlers: IsoChannelHandlers< ChannelMessage > = { @@ -833,13 +783,9 @@ function subscribeToHarnessContinuation( }; channel.subscribe(handlers); - return () => { - unbindParentStore(); - channel.unsubscribe(handlers); - }; } -function subscribeToAISDKV7TelemetryDispatcher(): () => void { +function subscribeToAISDKV7TelemetryDispatcher(): void { const channel = aiSDKChannels.v7CreateTelemetryDispatcher.tracingChannel(); const telemetry = braintrustAISDKTelemetry(); const handlers: IsoChannelHandlers< @@ -860,10 +806,6 @@ function subscribeToAISDKV7TelemetryDispatcher(): () => void { }; channel.subscribe(handlers); - - return () => { - channel.unsubscribe(handlers); - }; } function patchAISDKV7TelemetryDispatcher( @@ -4274,3 +4216,12 @@ function omit( return result; } + +let aiSDKInstrumentationConsumer: AISDKInstrumentationConsumer | undefined; + +export function registerAISDKInstrumentation( + config: AISDKInstrumentationConfig = {}, +): void { + aiSDKInstrumentationConsumer ??= new AISDKInstrumentationConsumer(config); + aiSDKInstrumentationConsumer.register(); +} diff --git a/js/src/instrumentation/plugins/ai-sdk-v7-telemetry.test.ts b/js/src/instrumentation/providers/ai-sdk-v7-telemetry.test.ts similarity index 100% rename from js/src/instrumentation/plugins/ai-sdk-v7-telemetry.test.ts rename to js/src/instrumentation/providers/ai-sdk-v7-telemetry.test.ts diff --git a/js/src/instrumentation/plugins/anthropic-channels.ts b/js/src/instrumentation/providers/anthropic-channels.ts similarity index 100% rename from js/src/instrumentation/plugins/anthropic-channels.ts rename to js/src/instrumentation/providers/anthropic-channels.ts diff --git a/js/src/instrumentation/plugins/anthropic-plugin.test.ts b/js/src/instrumentation/providers/anthropic-instrumentation.test.ts similarity index 99% rename from js/src/instrumentation/plugins/anthropic-plugin.test.ts rename to js/src/instrumentation/providers/anthropic-instrumentation.test.ts index ffad58aff..3e3d56708 100644 --- a/js/src/instrumentation/plugins/anthropic-plugin.test.ts +++ b/js/src/instrumentation/providers/anthropic-instrumentation.test.ts @@ -12,7 +12,7 @@ import { aggregateAnthropicStreamChunks, processAttachmentsInInput, coalesceInput, -} from "./anthropic-plugin"; +} from "./anthropic-instrumentation"; import { Attachment } from "../../logger"; const parseMetricsFromUsageForTest = (usage: unknown) => diff --git a/js/src/instrumentation/plugins/anthropic-plugin.ts b/js/src/instrumentation/providers/anthropic-instrumentation.ts similarity index 98% rename from js/src/instrumentation/plugins/anthropic-plugin.ts rename to js/src/instrumentation/providers/anthropic-instrumentation.ts index 36cad5a5c..9b6cc128e 100644 --- a/js/src/instrumentation/plugins/anthropic-plugin.ts +++ b/js/src/instrumentation/providers/anthropic-instrumentation.ts @@ -1,5 +1,5 @@ -import { BasePlugin, toLoggedError } from "../core"; -import { traceStreamingChannel, unsubscribeAll } from "../core/channel-tracing"; +import { toLoggedError } from "../core"; +import { traceStreamingChannel } from "../core/channel-tracing"; import { isAsyncIterable, patchStreamIfNeeded } from "../core/stream-patcher"; import { Attachment, @@ -93,30 +93,26 @@ const ANTHROPIC_TOOL_RUNNER_TOOL_WRAPPED = Symbol.for( ); /** - * Auto-instrumentation plugin for the Anthropic SDK. + * Internal auto-instrumentation consumer for the Anthropic SDK. * - * This plugin subscribes to orchestrion channels for Anthropic SDK methods + * This consumer subscribes to orchestrion channels for Anthropic SDK methods * and creates Braintrust spans to track: * - messages.create (streaming and non-streaming) * - beta.messages.create (streaming and non-streaming) * - * The plugin handles: + * The consumer handles: * - Anthropic-specific token metrics (including cache tokens) * - Processing message streams * - Converting base64 attachments to Attachment objects * - Streaming and non-streaming responses */ -export class AnthropicPlugin extends BasePlugin { - protected onEnable(): void { +class AnthropicInstrumentationConsumer { + public register(): void { this.subscribeToAnthropicChannels(); this.subscribeToAnthropicToolRunner(); this.subscribeToAnthropicSessionStreams(); } - protected onDisable(): void { - this.unsubscribers = unsubscribeAll(this.unsubscribers); - } - private subscribeToAnthropicChannels(): void { const anthropicConfig = { name: "anthropic.messages.create", @@ -163,17 +159,13 @@ export class AnthropicPlugin extends BasePlugin { }; // Messages API - supports streaming via stream=true parameter - this.unsubscribers.push( - traceStreamingChannel(anthropicChannels.messagesCreate, anthropicConfig), - ); + traceStreamingChannel(anthropicChannels.messagesCreate, anthropicConfig); // Beta Messages API - supports streaming via stream=true parameter - this.unsubscribers.push( - traceStreamingChannel(anthropicChannels.betaMessagesCreate, { - ...anthropicConfig, - name: "anthropic.messages.create", - }), - ); + traceStreamingChannel(anthropicChannels.betaMessagesCreate, { + ...anthropicConfig, + name: "anthropic.messages.create", + }); } private subscribeToAnthropicToolRunner(): void { @@ -250,9 +242,6 @@ export class AnthropicPlugin extends BasePlugin { }; tracingChannel.subscribe(handlers); - this.unsubscribers.push(() => { - tracingChannel.unsubscribe(handlers); - }); } private subscribeToAnthropicSessionStreams(): void { @@ -305,7 +294,6 @@ export class AnthropicPlugin extends BasePlugin { }; tracingChannel.subscribe(handlers); - this.unsubscribers.push(() => tracingChannel.unsubscribe(handlers)); } } @@ -1686,3 +1674,12 @@ export function coalesceInput( } return input; } + +let anthropicInstrumentationConsumer: + | AnthropicInstrumentationConsumer + | undefined; + +export function registerAnthropicInstrumentation(): void { + anthropicInstrumentationConsumer ??= new AnthropicInstrumentationConsumer(); + anthropicInstrumentationConsumer.register(); +} diff --git a/js/src/instrumentation/plugins/anthropic-sessions-plugin.test.ts b/js/src/instrumentation/providers/anthropic-sessions-instrumentation.test.ts similarity index 97% rename from js/src/instrumentation/plugins/anthropic-sessions-plugin.test.ts rename to js/src/instrumentation/providers/anthropic-sessions-instrumentation.test.ts index 1d4cb48ac..44bd501d6 100644 --- a/js/src/instrumentation/plugins/anthropic-sessions-plugin.test.ts +++ b/js/src/instrumentation/providers/anthropic-sessions-instrumentation.test.ts @@ -28,11 +28,11 @@ vi.mock("../../logger", async (importOriginal) => { import iso from "../../isomorph"; import { collectAnthropicSession } from "../../wrappers/anthropic-session-collector"; -import { AnthropicPlugin } from "./anthropic-plugin"; +import { registerAnthropicInstrumentation } from "./anthropic-instrumentation"; const mockNewTracingChannel = iso.newTracingChannel as ReturnType; -describe("AnthropicPlugin Sessions instrumentation", () => { +describe("registerAnthropicInstrumentation Sessions instrumentation", () => { let currentSpan: TestSpan | undefined; let handlersByName: Map; let spans: TestSpan[]; @@ -43,7 +43,6 @@ describe("AnthropicPlugin Sessions instrumentation", () => { spans = []; mockNewTracingChannel.mockImplementation((name: string) => ({ subscribe: vi.fn((handlers) => handlersByName.set(name, handlers)), - unsubscribe: vi.fn(), })); mockWithCurrent.mockImplementation( (span: TestSpan, callback: () => unknown) => { @@ -78,7 +77,7 @@ describe("AnthropicPlugin Sessions instrumentation", () => { }); it("records a session turn with model and tool children", async () => { - new AnthropicPlugin().enable(); + registerAnthropicInstrumentation(); const handlers = handlersByName.get( "orchestrion:@anthropic-ai/sdk:beta.sessions.events.stream", ); @@ -262,7 +261,7 @@ describe("AnthropicPlugin Sessions instrumentation", () => { }); it("does not record an uncollected session stream", async () => { - new AnthropicPlugin().enable(); + registerAnthropicInstrumentation(); const handlers = handlersByName.get( "orchestrion:@anthropic-ai/sdk:beta.sessions.events.stream", ); @@ -286,7 +285,7 @@ describe("AnthropicPlugin Sessions instrumentation", () => { }); it("uses a distinct task name for thread streams and records denials", async () => { - new AnthropicPlugin().enable(); + registerAnthropicInstrumentation(); const handlers = handlersByName.get( "orchestrion:@anthropic-ai/sdk:beta.sessions.threads.events.stream", ); diff --git a/js/src/instrumentation/plugins/bedrock-runtime-channels.ts b/js/src/instrumentation/providers/bedrock-runtime-channels.ts similarity index 100% rename from js/src/instrumentation/plugins/bedrock-runtime-channels.ts rename to js/src/instrumentation/providers/bedrock-runtime-channels.ts diff --git a/js/src/instrumentation/plugins/bedrock-runtime-common.ts b/js/src/instrumentation/providers/bedrock-runtime-common.ts similarity index 100% rename from js/src/instrumentation/plugins/bedrock-runtime-common.ts rename to js/src/instrumentation/providers/bedrock-runtime-common.ts diff --git a/js/src/instrumentation/plugins/bedrock-runtime-plugin.test.ts b/js/src/instrumentation/providers/bedrock-runtime-instrumentation.test.ts similarity index 99% rename from js/src/instrumentation/plugins/bedrock-runtime-plugin.test.ts rename to js/src/instrumentation/providers/bedrock-runtime-instrumentation.test.ts index fc74a9068..b22906083 100644 --- a/js/src/instrumentation/plugins/bedrock-runtime-plugin.test.ts +++ b/js/src/instrumentation/providers/bedrock-runtime-instrumentation.test.ts @@ -9,7 +9,7 @@ import { aggregateBedrockConverseStreamChunks, aggregateInvokeModelResponseStreamChunks, parseBedrockRuntimeMetrics, -} from "./bedrock-runtime-plugin"; +} from "./bedrock-runtime-instrumentation"; try { configureNode(); @@ -29,7 +29,7 @@ class GetObjectCommand { constructor(public input: Record) {} } -describe("BedrockRuntimePlugin", () => { +describe("registerBedrockRuntimeInstrumentation", () => { let backgroundLogger: ReturnType< typeof _exportsForTestingOnly.useTestBackgroundLogger >; @@ -41,7 +41,7 @@ describe("BedrockRuntimePlugin", () => { beforeEach(() => { backgroundLogger = _exportsForTestingOnly.useTestBackgroundLogger(); initLogger({ - projectName: "bedrock-runtime-plugin.test.ts", + projectName: "bedrock-runtime-instrumentation.test.ts", projectId: "test-project-id", }); }); diff --git a/js/src/instrumentation/plugins/bedrock-runtime-plugin.ts b/js/src/instrumentation/providers/bedrock-runtime-instrumentation.ts similarity index 97% rename from js/src/instrumentation/plugins/bedrock-runtime-plugin.ts rename to js/src/instrumentation/providers/bedrock-runtime-instrumentation.ts index a759d1ffe..dfd5a47be 100644 --- a/js/src/instrumentation/plugins/bedrock-runtime-plugin.ts +++ b/js/src/instrumentation/providers/bedrock-runtime-instrumentation.ts @@ -1,5 +1,4 @@ -import { BasePlugin } from "../core"; -import { traceStreamingChannel, unsubscribeAll } from "../core/channel-tracing"; +import { traceStreamingChannel } from "../core/channel-tracing"; import { isAsyncIterable, patchStreamIfNeeded } from "../core/stream-patcher"; import { SpanTypeAttribute, isObject } from "../../../util/index"; import { getCurrentUnixTimestamp } from "../../util"; @@ -25,26 +24,18 @@ import { getBedrockRuntimeOperation, } from "./bedrock-runtime-common"; -export class BedrockRuntimePlugin extends BasePlugin { - protected onEnable(): void { - this.unsubscribers.push( - ...[ - bedrockRuntimeChannels.clientSend, - smithyCoreChannels.clientSend, - smithyClientChannels.clientSend, - ].map((channel) => traceBedrockRuntimeClientSendChannel(channel)), - ); - } - - protected onDisable(): void { - this.unsubscribers = unsubscribeAll(this.unsubscribers); +export function registerBedrockRuntimeInstrumentation(): void { + for (const channel of [ + bedrockRuntimeChannels.clientSend, + smithyCoreChannels.clientSend, + smithyClientChannels.clientSend, + ]) { + traceBedrockRuntimeClientSendChannel(channel); } } -function traceBedrockRuntimeClientSendChannel( - channel: AnyAsyncChannel, -): () => void { - return traceStreamingChannel(channel, { +function traceBedrockRuntimeClientSendChannel(channel: AnyAsyncChannel): void { + traceStreamingChannel(channel, { name: ([command]) => buildBedrockRuntimeSpanInfo(command).name, shouldTrace: ([command, optionsOrCb, cb]) => getBedrockRuntimeOperation(command) !== undefined && diff --git a/js/src/instrumentation/plugins/claude-agent-sdk-channels.ts b/js/src/instrumentation/providers/claude-agent-sdk-channels.ts similarity index 100% rename from js/src/instrumentation/plugins/claude-agent-sdk-channels.ts rename to js/src/instrumentation/providers/claude-agent-sdk-channels.ts diff --git a/js/src/instrumentation/plugins/claude-agent-sdk-instrumentation-constants.ts b/js/src/instrumentation/providers/claude-agent-sdk-instrumentation-constants.ts similarity index 100% rename from js/src/instrumentation/plugins/claude-agent-sdk-instrumentation-constants.ts rename to js/src/instrumentation/providers/claude-agent-sdk-instrumentation-constants.ts diff --git a/js/src/instrumentation/plugins/claude-agent-sdk-plugin.streaming.test.ts b/js/src/instrumentation/providers/claude-agent-sdk-instrumentation.streaming.test.ts similarity index 99% rename from js/src/instrumentation/plugins/claude-agent-sdk-plugin.streaming.test.ts rename to js/src/instrumentation/providers/claude-agent-sdk-instrumentation.streaming.test.ts index 42101e066..ddaeeab1f 100644 --- a/js/src/instrumentation/plugins/claude-agent-sdk-plugin.streaming.test.ts +++ b/js/src/instrumentation/providers/claude-agent-sdk-instrumentation.streaming.test.ts @@ -120,7 +120,7 @@ describe("Claude Agent SDK streaming instrumentation", () => { backgroundLogger = _exportsForTestingOnly.useTestBackgroundLogger(); initLogger({ projectId: "test-project-id", - projectName: "claude-agent-sdk-plugin.streaming.test.ts", + projectName: "claude-agent-sdk-instrumentation.streaming.test.ts", }); }); diff --git a/js/src/instrumentation/plugins/claude-agent-sdk-plugin.test.ts b/js/src/instrumentation/providers/claude-agent-sdk-instrumentation.test.ts similarity index 86% rename from js/src/instrumentation/plugins/claude-agent-sdk-plugin.test.ts rename to js/src/instrumentation/providers/claude-agent-sdk-instrumentation.test.ts index f1d446bcd..93265e4a3 100644 --- a/js/src/instrumentation/plugins/claude-agent-sdk-plugin.test.ts +++ b/js/src/instrumentation/providers/claude-agent-sdk-instrumentation.test.ts @@ -30,7 +30,7 @@ vi.mock("../core/stream-patcher", () => ({ }), })); -import { ClaudeAgentSDKPlugin } from "./claude-agent-sdk-plugin"; +import { registerClaudeAgentSDKInstrumentation } from "./claude-agent-sdk-instrumentation"; import iso from "../../isomorph"; import { startSpan } from "../../logger"; @@ -72,79 +72,26 @@ vi.mock("../../wrappers/attachment-utils", () => ({ // rules, so these tests run the real implementation rather than a stand-in that // could drift from it. -vi.mock("../core", async (importOriginal) => { - const actual = await importOriginal(); - - return { - ...actual, - BasePlugin: class BasePlugin { - protected enabled = false; - protected unsubscribers: Array<() => void> = []; - - enable(): void { - if (this.enabled) { - return; - } - this.enabled = true; - this.onEnable(); - } - - disable(): void { - if (!this.enabled) { - return; - } - this.enabled = false; - this.onDisable(); - } - - protected onEnable(): void { - // To be implemented by subclass - } - - protected onDisable(): void { - // To be implemented by subclass - } - }, - isAsyncIterable: vi.fn( - (val: unknown) => - val !== null && - typeof val === "object" && - Symbol.asyncIterator in val && - typeof (val as any)[Symbol.asyncIterator] === "function", - ), - patchStreamIfNeeded: vi.fn((stream, _callbacks) => { - // Return the stream unchanged for simple tests - return stream; - }), - }; -}); - -describe("ClaudeAgentSDKPlugin", () => { - let plugin: ClaudeAgentSDKPlugin; +describe("registerClaudeAgentSDKInstrumentation", () => { let mockChannel: any; - let mockUnsubscribe: any; beforeEach(() => { streamPatcherMock.options = undefined; - mockUnsubscribe = vi.fn(); mockChannel = { subscribe: vi.fn(), - unsubscribe: mockUnsubscribe, hasSubscribers: false, }; mockNewTracingChannel.mockReturnValue(mockChannel); - - plugin = new ClaudeAgentSDKPlugin(); }); afterEach(() => { vi.clearAllMocks(); }); - describe("enable", () => { - it("should enable the plugin and subscribe to channels", () => { - plugin.enable(); + describe("registration", () => { + it("subscribes to the channel", () => { + registerClaudeAgentSDKInstrumentation(); expect(mockNewTracingChannel).toHaveBeenCalledWith( "orchestrion:@anthropic-ai/claude-agent-sdk:query", @@ -158,50 +105,13 @@ describe("ClaudeAgentSDKPlugin", () => { }), ); }); - - it("should not subscribe twice if already enabled", () => { - plugin.enable(); - plugin.enable(); - - expect(mockChannel.subscribe).toHaveBeenCalledTimes(1); - }); - - it("should store unsubscribe function", () => { - plugin.enable(); - - expect((plugin as any).unsubscribers).toHaveLength(1); - expect((plugin as any).unsubscribers[0]).toBeInstanceOf(Function); - }); - }); - - describe("disable", () => { - it("should unsubscribe from all channels", () => { - plugin.enable(); - plugin.disable(); - - expect(mockUnsubscribe).toHaveBeenCalledTimes(1); - expect((plugin as any).unsubscribers).toHaveLength(0); - }); - - it("should not unsubscribe if not enabled", () => { - plugin.disable(); - - expect(mockUnsubscribe).not.toHaveBeenCalled(); - }); - - it("should clear unsubscribers array", () => { - plugin.enable(); - plugin.disable(); - - expect((plugin as any).unsubscribers).toHaveLength(0); - }); }); describe("channel subscription handlers", () => { let handlers: any; beforeEach(() => { - plugin.enable(); + registerClaudeAgentSDKInstrumentation(); handlers = mockChannel.subscribe.mock.calls[0][0]; }); @@ -745,24 +655,4 @@ describe("ClaudeAgentSDKPlugin", () => { }); }); }); - - describe("enable/disable lifecycle", () => { - it("should allow re-enabling after disable", () => { - plugin.enable(); - plugin.disable(); - plugin.enable(); - - expect(mockChannel.subscribe).toHaveBeenCalledTimes(2); - }); - - it("should properly clean up on multiple enable/disable cycles", () => { - plugin.enable(); - plugin.disable(); - plugin.enable(); - plugin.disable(); - - expect(mockUnsubscribe).toHaveBeenCalledTimes(2); - expect((plugin as any).unsubscribers).toHaveLength(0); - }); - }); }); diff --git a/js/src/instrumentation/plugins/claude-agent-sdk-plugin.ts b/js/src/instrumentation/providers/claude-agent-sdk-instrumentation.ts similarity index 99% rename from js/src/instrumentation/plugins/claude-agent-sdk-plugin.ts rename to js/src/instrumentation/providers/claude-agent-sdk-instrumentation.ts index 96f52c9b6..9944ebe60 100644 --- a/js/src/instrumentation/plugins/claude-agent-sdk-plugin.ts +++ b/js/src/instrumentation/providers/claude-agent-sdk-instrumentation.ts @@ -1,4 +1,3 @@ -import { BasePlugin } from "../core"; import type { ChannelMessage } from "../core/channel-definitions"; import { isAsyncIterable, patchStreamIfNeeded } from "../core/stream-patcher"; import type { IsoChannelHandlers } from "../../isomorph"; @@ -1578,18 +1577,11 @@ async function finalizeQuerySpan(state: QueryState): Promise { } } -export class ClaudeAgentSDKPlugin extends BasePlugin { - protected onEnable(): void { +class ClaudeAgentSDKInstrumentationConsumer { + public register(): void { this.subscribeToQuery(); } - protected onDisable(): void { - for (const unsubscribe of this.unsubscribers) { - unsubscribe(); - } - this.unsubscribers = []; - } - private subscribeToQuery(): void { const channel = claudeAgentSDKChannels.query.tracingChannel(); const spans = new WeakMap(); @@ -1854,8 +1846,15 @@ export class ClaudeAgentSDKPlugin extends BasePlugin { }; channel.subscribe(handlers); - this.unsubscribers.push(() => { - channel.unsubscribe(handlers); - }); } } + +let claudeAgentSDKInstrumentationConsumer: + | ClaudeAgentSDKInstrumentationConsumer + | undefined; + +export function registerClaudeAgentSDKInstrumentation(): void { + claudeAgentSDKInstrumentationConsumer ??= + new ClaudeAgentSDKInstrumentationConsumer(); + claudeAgentSDKInstrumentationConsumer.register(); +} diff --git a/js/src/instrumentation/plugins/claude-agent-sdk-local-tool-context.ts b/js/src/instrumentation/providers/claude-agent-sdk-local-tool-context.ts similarity index 100% rename from js/src/instrumentation/plugins/claude-agent-sdk-local-tool-context.ts rename to js/src/instrumentation/providers/claude-agent-sdk-local-tool-context.ts diff --git a/js/src/instrumentation/plugins/claude-agent-sdk-local-tool-spans.ts b/js/src/instrumentation/providers/claude-agent-sdk-local-tool-spans.ts similarity index 100% rename from js/src/instrumentation/plugins/claude-agent-sdk-local-tool-spans.ts rename to js/src/instrumentation/providers/claude-agent-sdk-local-tool-spans.ts diff --git a/js/src/instrumentation/plugins/cloudflare-agents-channels.ts b/js/src/instrumentation/providers/cloudflare-agents-channels.ts similarity index 100% rename from js/src/instrumentation/plugins/cloudflare-agents-channels.ts rename to js/src/instrumentation/providers/cloudflare-agents-channels.ts diff --git a/js/src/instrumentation/plugins/cloudflare-agents-plugin.test.ts b/js/src/instrumentation/providers/cloudflare-agents-instrumentation.test.ts similarity index 88% rename from js/src/instrumentation/plugins/cloudflare-agents-plugin.test.ts rename to js/src/instrumentation/providers/cloudflare-agents-instrumentation.test.ts index d6d2de7de..970297b01 100644 --- a/js/src/instrumentation/plugins/cloudflare-agents-plugin.test.ts +++ b/js/src/instrumentation/providers/cloudflare-agents-instrumentation.test.ts @@ -29,14 +29,13 @@ vi.mock("../../isomorph", () => ({ })); import iso from "../../isomorph"; -import { CloudflareAgentsPlugin } from "./cloudflare-agents-plugin"; +import { registerCloudflareAgentsInstrumentation } from "./cloudflare-agents-instrumentation"; const mockNewTracingChannel = iso.newTracingChannel as ReturnType; -describe("CloudflareAgentsPlugin", () => { +describe("registerCloudflareAgentsInstrumentation", () => { let handlers: any; let subscribe: ReturnType; - let unsubscribe: ReturnType; let spans: Array<{ args: any; context: any; @@ -49,8 +48,7 @@ describe("CloudflareAgentsPlugin", () => { subscribe = vi.fn((nextHandlers) => { handlers = nextHandlers; }); - unsubscribe = vi.fn(); - mockNewTracingChannel.mockReturnValue({ subscribe, unsubscribe }); + mockNewTracingChannel.mockReturnValue({ subscribe }); mockStartSpan.mockImplementation((args: any, context: any) => { const span = { args, context, end: vi.fn(), log: vi.fn() }; spans.push(span); @@ -62,19 +60,13 @@ describe("CloudflareAgentsPlugin", () => { vi.clearAllMocks(); }); - it("subscribes idempotently to Agent.runAgentTool", () => { - const plugin = new CloudflareAgentsPlugin(); - plugin.enable(); - plugin.enable(); + it("subscribes to the process-lifetime channel", () => { + registerCloudflareAgentsInstrumentation(); expect(mockNewTracingChannel).toHaveBeenCalledWith( "orchestrion:agents:Agent.runAgentTool", ); expect(subscribe).toHaveBeenCalledTimes(1); - - plugin.disable(); - plugin.disable(); - expect(unsubscribe).toHaveBeenCalledTimes(1); }); it("keeps SDK-controlled context out of the public start-span arguments", () => { @@ -86,7 +78,7 @@ describe("CloudflareAgentsPlugin", () => { }); it("records only the child class name, input, and completed output", () => { - new CloudflareAgentsPlugin().enable(); + registerCloudflareAgentsInstrumentation(); class ResearchAgent {} const event = { arguments: [ @@ -142,7 +134,7 @@ describe("CloudflareAgentsPlugin", () => { }); it("records returned terminal error strings", () => { - new CloudflareAgentsPlugin().enable(); + registerCloudflareAgentsInstrumentation(); class FailingAgent {} const event = { arguments: [FailingAgent, { input: "fail" }] }; @@ -164,7 +156,7 @@ describe("CloudflareAgentsPlugin", () => { }); it("records the original rejection and preserves concurrent span state", () => { - new CloudflareAgentsPlugin().enable(); + registerCloudflareAgentsInstrumentation(); class FirstAgent {} class SecondAgent {} const first = { arguments: [FirstAgent, { input: 1 }] }; @@ -188,7 +180,7 @@ describe("CloudflareAgentsPlugin", () => { }); it("skips detached runs and does not invoke getters", () => { - new CloudflareAgentsPlugin().enable(); + registerCloudflareAgentsInstrumentation(); const nameGetter = vi.fn(() => "GetterAgent"); const inputGetter = vi.fn(() => "getter-input"); const AgentWithGetter = Object.defineProperty(function () {}, "name", { diff --git a/js/src/instrumentation/providers/cloudflare-agents-instrumentation.ts b/js/src/instrumentation/providers/cloudflare-agents-instrumentation.ts new file mode 100644 index 000000000..36fe3671a --- /dev/null +++ b/js/src/instrumentation/providers/cloudflare-agents-instrumentation.ts @@ -0,0 +1,129 @@ +import { debugLogger } from "../../debug-logger"; +import type { IsoChannelHandlers } from "../../isomorph"; +import { _internalStartSpanWithContext } from "../../logger"; +import type { Span } from "../../logger"; +import { + INSTRUMENTATION_NAMES, + withSpanInstrumentationName, +} from "../../span-origin"; +import { SpanTypeAttribute } from "../../../util/index"; +import type { ChannelMessage } from "../core/channel-definitions"; +import { cloudflareAgentsChannels } from "./cloudflare-agents-channels"; + +const CLOUDFLARE_WORKERS_CONTEXT = { + span_origin: { + environment: { type: "server", name: "cloudflare_workers" }, + }, +}; + +export function registerCloudflareAgentsInstrumentation(): void { + const channel = cloudflareAgentsChannels.runAgentTool.tracingChannel(); + const spans = new WeakMap(); + const handlers: IsoChannelHandlers< + ChannelMessage + > = { + start: (event) => { + try { + const agentClass = event.arguments[0]; + const options = event.arguments[1]; + if (ownValue(options, "detached")) { + return; + } + + const name = ownValue(agentClass, "name"); + if (typeof name !== "string" || name.length === 0) { + debugLogger.warn( + "Skipping Cloudflare Agents runAgentTool span because the child agent class has no name.", + ); + return; + } + + const span = _internalStartSpanWithContext( + withSpanInstrumentationName( + { + name, + spanAttributes: { type: SpanTypeAttribute.TOOL }, + event: { + input: ownValue(options, "input"), + }, + }, + INSTRUMENTATION_NAMES.CLOUDFLARE_AGENTS, + ), + CLOUDFLARE_WORKERS_CONTEXT, + ); + spans.set(event, span); + } catch (error) { + logInstrumentationError("start", error); + } + }, + asyncEnd: (event) => { + const span = spans.get(event); + if (!span) { + return; + } + spans.delete(event); + + try { + const status = ownValue(event.result, "status"); + if (status === "completed") { + span.log({ output: ownValue(event.result, "output") }); + } else { + const error = ownValue(event.result, "error"); + if (typeof error === "string") { + span.log({ error }); + } + } + } catch (error) { + logInstrumentationError("completion", error); + } finally { + safelyEndSpan(span); + } + }, + error: (event) => { + const span = spans.get(event); + if (!span) { + return; + } + spans.delete(event); + + try { + span.log({ error: event.error }); + } catch (error) { + logInstrumentationError("rejection", error); + } finally { + safelyEndSpan(span); + } + }, + }; + + channel.subscribe(handlers); +} + +function ownValue(value: unknown, key: PropertyKey): unknown { + if (!isObjectLike(value)) { + return undefined; + } + const descriptor = Object.getOwnPropertyDescriptor(value, key); + return descriptor && "value" in descriptor ? descriptor.value : undefined; +} + +function isObjectLike(value: unknown): value is object { + return ( + (typeof value === "object" && value !== null) || typeof value === "function" + ); +} + +function safelyEndSpan(span: Span): void { + try { + span.end(); + } catch (error) { + logInstrumentationError("span end", error); + } +} + +function logInstrumentationError(operation: string, error: unknown): void { + debugLogger.error( + `Failed to process Cloudflare Agents ${operation} instrumentation:`, + error, + ); +} diff --git a/js/src/instrumentation/plugins/cloudflare-ai-chat-channels.ts b/js/src/instrumentation/providers/cloudflare-ai-chat-channels.ts similarity index 100% rename from js/src/instrumentation/plugins/cloudflare-ai-chat-channels.ts rename to js/src/instrumentation/providers/cloudflare-ai-chat-channels.ts diff --git a/js/src/instrumentation/plugins/cloudflare-ai-chat-plugin.test.ts b/js/src/instrumentation/providers/cloudflare-ai-chat-consumer.test.ts similarity index 94% rename from js/src/instrumentation/plugins/cloudflare-ai-chat-plugin.test.ts rename to js/src/instrumentation/providers/cloudflare-ai-chat-consumer.test.ts index d20631a2d..860ab0b18 100644 --- a/js/src/instrumentation/plugins/cloudflare-ai-chat-plugin.test.ts +++ b/js/src/instrumentation/providers/cloudflare-ai-chat-consumer.test.ts @@ -30,12 +30,11 @@ import { INSTRUMENTATION_NAMES, INTERNAL_SPAN_INSTRUMENTATION_NAME, } from "../../span-origin"; -import { CloudflareAIChatPlugin } from "./cloudflare-ai-chat-plugin"; +import { registerCloudflareAIChatInstrumentation } from "./cloudflare-ai-chat-consumer"; const mockNewTracingChannel = iso.newTracingChannel as ReturnType; -describe("CloudflareAIChatPlugin", () => { - let plugin: CloudflareAIChatPlugin; +describe("registerCloudflareAIChatInstrumentation", () => { let channels: Map>; beforeEach(() => { @@ -54,16 +53,14 @@ describe("CloudflareAIChatPlugin", () => { log: vi.fn(), })); mockInternalGetGlobalState.mockReturnValue(undefined); - plugin = new CloudflareAIChatPlugin(); }); afterEach(() => { - plugin.disable(); vi.clearAllMocks(); }); it("captures the full successful turn and binds queued work", async () => { - plugin.enable(); + registerCloudflareAIChatInstrumentation(); const turnHandlers = turnChannel().handlers(); const callback = vi.fn(async () => "callback-result"); const agent = { @@ -132,7 +129,7 @@ describe("CloudflareAIChatPlugin", () => { }); it("correlates response errors and preserves partial output", async () => { - plugin.enable(); + registerCloudflareAIChatInstrumentation(); const turnHandlers = turnChannel().handlers(); const responseHandlers = responseChannel().handlers(); const agent = { messages: [], onChatResponse() {} }; @@ -169,7 +166,7 @@ describe("CloudflareAIChatPlugin", () => { }); it("retains a settled turn until its queued response is observed", async () => { - plugin.enable(); + registerCloudflareAIChatInstrumentation(); const handlers = turnChannel().handlers(); const agent = { messages: [ @@ -225,7 +222,7 @@ describe("CloudflareAIChatPlugin", () => { it("drops retained turns that never produce a response", () => { vi.useFakeTimers(); try { - plugin.enable(); + registerCloudflareAIChatInstrumentation(); const handlers = turnChannel().handlers(); const agent = { messages: [], @@ -263,7 +260,7 @@ describe("CloudflareAIChatPlugin", () => { }); it("preserves pre-turn input when a continuation reuses its output id", async () => { - plugin.enable(); + registerCloudflareAIChatInstrumentation(); const handlers = turnChannel().handlers(); const agent = { messages: [ @@ -324,7 +321,7 @@ describe("CloudflareAIChatPlugin", () => { }); it("deduplicates nested manual and automatic turn events", () => { - plugin.enable(); + registerCloudflareAIChatInstrumentation(); const handlers = turnChannel().handlers(); const agent = { messages: [], onChatResponse() {} }; const outer = { @@ -348,8 +345,8 @@ describe("CloudflareAIChatPlugin", () => { expect(span.end).toHaveBeenCalledTimes(1); }); - it("logs original errors and closes outstanding spans on disable", () => { - plugin.enable(); + it("logs original errors without closing unrelated outstanding spans", () => { + registerCloudflareAIChatInstrumentation(); const handlers = turnChannel().handlers(); const failure = new Error("turn failed"); const failedEvent = { @@ -370,8 +367,7 @@ describe("CloudflareAIChatPlugin", () => { } as any; handlers.start?.(pendingEvent, "start"); const pendingSpan = mockStartSpan.mock.results[1].value; - plugin.disable(); - expect(pendingSpan.end).toHaveBeenCalledTimes(1); + expect(pendingSpan.end).not.toHaveBeenCalled(); }); function turnChannel() { @@ -394,7 +390,6 @@ function createMockChannel() { hasSubscribers: false, start: { bindStore: vi.fn(), - unbindStore: vi.fn(), }, subscribe: vi.fn((handlers) => subscribed.push(handlers)), traceSync: vi.fn((callback, event) => { @@ -410,6 +405,5 @@ function createMockChannel() { throw error; } }), - unsubscribe: vi.fn(), }; } diff --git a/js/src/instrumentation/plugins/cloudflare-ai-chat-plugin.ts b/js/src/instrumentation/providers/cloudflare-ai-chat-consumer.ts similarity index 94% rename from js/src/instrumentation/plugins/cloudflare-ai-chat-plugin.ts rename to js/src/instrumentation/providers/cloudflare-ai-chat-consumer.ts index d875824af..a75ad3ea6 100644 --- a/js/src/instrumentation/plugins/cloudflare-ai-chat-plugin.ts +++ b/js/src/instrumentation/providers/cloudflare-ai-chat-consumer.ts @@ -1,4 +1,3 @@ -import { BasePlugin } from "../core"; import type { ChannelMessage } from "../core/channel-definitions"; import type { IsoChannelHandlers, IsoTracingChannel } from "../../isomorph"; import { @@ -41,7 +40,7 @@ type TurnState = { const BLOCKED_KEYS = new Set(["__proto__", "constructor", "prototype"]); const RESPONSE_STATE_RETENTION_MS = 60_000; -export class CloudflareAIChatPlugin extends BasePlugin { +class CloudflareAIChatInstrumentationConsumer { private readonly activeStates = new Set(); private readonly activeTurns = new WeakMap< object, @@ -49,30 +48,18 @@ export class CloudflareAIChatPlugin extends BasePlugin { >(); private readonly eventStates = new WeakMap(); - protected onEnable(): void { + public register(): void { this.subscribeToResponseHook(); this.subscribeToTurnRunner(); } - protected onDisable(): void { - for (const unsubscribe of this.unsubscribers) { - unsubscribe(); - } - this.unsubscribers = []; - - for (const state of this.activeStates) { - this.cleanupState(state); - } - this.activeStates.clear(); - } - private subscribeToTurnRunner(): void { const tracingChannel = cloudflareAIChatChannels.runExclusiveChatTurn.tracingChannel() as IsoTracingChannel< ChannelMessage >; - const unbindCurrentSpanStore = this.bindCurrentSpanStore(tracingChannel); + this.bindCurrentSpanStore(tracingChannel); const handlers: IsoChannelHandlers> = { start: (event) => { this.ensureEventState(event); @@ -86,10 +73,6 @@ export class CloudflareAIChatPlugin extends BasePlugin { }; tracingChannel.subscribe(handlers); - this.unsubscribers.push(() => { - unbindCurrentSpanStore?.(); - tracingChannel.unsubscribe(handlers); - }); } private subscribeToResponseHook(): void { @@ -143,12 +126,11 @@ export class CloudflareAIChatPlugin extends BasePlugin { }; tracingChannel.subscribe(handlers); - this.unsubscribers.push(() => tracingChannel.unsubscribe(handlers)); } private bindCurrentSpanStore( tracingChannel: IsoTracingChannel>, - ): (() => void) | undefined { + ): void { const globalState = _internalGetGlobalState(); const contextManager = globalState?.contextManager; const startChannel = tracingChannel.start; @@ -161,7 +143,7 @@ export class CloudflareAIChatPlugin extends BasePlugin { : undefined; if (!startChannel || !currentSpanStore || !contextManager) { - return undefined; + return; } startChannel.bindStore(currentSpanStore, (event) => { @@ -170,8 +152,6 @@ export class CloudflareAIChatPlugin extends BasePlugin { ? contextManager.wrapSpanForStore(state.span) : currentSpanStore.getStore(); }); - - return () => startChannel.unbindStore(currentSpanStore); } private ensureEventState( @@ -481,3 +461,13 @@ function sanitizeLoggedValue( } return Object.fromEntries(entries); } + +let cloudflareAIChatInstrumentationConsumer: + | CloudflareAIChatInstrumentationConsumer + | undefined; + +export function registerCloudflareAIChatInstrumentation(): void { + cloudflareAIChatInstrumentationConsumer ??= + new CloudflareAIChatInstrumentationConsumer(); + cloudflareAIChatInstrumentationConsumer.register(); +} diff --git a/js/src/instrumentation/plugins/cloudflare-ai-chat-instrumentation.ts b/js/src/instrumentation/providers/cloudflare-ai-chat-instrumentation.ts similarity index 100% rename from js/src/instrumentation/plugins/cloudflare-ai-chat-instrumentation.ts rename to js/src/instrumentation/providers/cloudflare-ai-chat-instrumentation.ts diff --git a/js/src/instrumentation/plugins/cloudflare-think-channels.ts b/js/src/instrumentation/providers/cloudflare-think-channels.ts similarity index 100% rename from js/src/instrumentation/plugins/cloudflare-think-channels.ts rename to js/src/instrumentation/providers/cloudflare-think-channels.ts diff --git a/js/src/instrumentation/plugins/cloudflare-think-context.ts b/js/src/instrumentation/providers/cloudflare-think-context.ts similarity index 100% rename from js/src/instrumentation/plugins/cloudflare-think-context.ts rename to js/src/instrumentation/providers/cloudflare-think-context.ts diff --git a/js/src/instrumentation/plugins/cloudflare-think-plugin.test.ts b/js/src/instrumentation/providers/cloudflare-think-instrumentation.test.ts similarity index 98% rename from js/src/instrumentation/plugins/cloudflare-think-plugin.test.ts rename to js/src/instrumentation/providers/cloudflare-think-instrumentation.test.ts index 240c33df0..f3e27919e 100644 --- a/js/src/instrumentation/plugins/cloudflare-think-plugin.test.ts +++ b/js/src/instrumentation/providers/cloudflare-think-instrumentation.test.ts @@ -27,7 +27,7 @@ describe("Cloudflare Think instrumentation", () => { beforeEach(() => { backgroundLogger = _exportsForTestingOnly.useTestBackgroundLogger(); initLogger({ - projectName: "cloudflare-think-plugin.test.ts", + projectName: "cloudflare-think-instrumentation.test.ts", projectId: "test-project-id", }); }); diff --git a/js/src/instrumentation/plugins/cloudflare-think-plugin.ts b/js/src/instrumentation/providers/cloudflare-think-instrumentation.ts similarity index 93% rename from js/src/instrumentation/plugins/cloudflare-think-plugin.ts rename to js/src/instrumentation/providers/cloudflare-think-instrumentation.ts index 8a29eb16c..66196becf 100644 --- a/js/src/instrumentation/plugins/cloudflare-think-plugin.ts +++ b/js/src/instrumentation/providers/cloudflare-think-instrumentation.ts @@ -1,4 +1,3 @@ -import { BasePlugin } from "../core"; import type { ChannelMessage } from "../core/channel-definitions"; import type { IsoChannelHandlers } from "../../isomorph"; import { @@ -25,7 +24,7 @@ import { finalizeAISDKChildTracing, patchAISDKStreamingResult, prepareAISDKAgentCallInput, -} from "./ai-sdk-plugin"; +} from "./ai-sdk-instrumentation"; import { cloudflareThinkChannels } from "./cloudflare-think-channels"; import { registerCloudflareThinkSpan, @@ -50,27 +49,15 @@ type AISDKStreamEvent = const THINK_STATE_ID = Symbol.for("braintrust.cloudflare-think.state-id"); -export class CloudflareThinkPlugin extends BasePlugin { +class CloudflareThinkInstrumentationConsumer { private readonly statesBySpanId = new Map(); - protected onEnable(): void { + public register(): void { this.subscribeToThinkRuns(); this.subscribeToAISDKStreamTextSync(); this.subscribeToAISDKStreamTextAsync(); } - protected onDisable(): void { - for (const unsubscribe of this.unsubscribers) { - unsubscribe(); - } - this.unsubscribers = []; - - for (const state of this.statesBySpanId.values()) { - this.finishState(state); - } - this.statesBySpanId.clear(); - } - private subscribeToThinkRuns(): void { const channel = cloudflareThinkChannels.runInferenceLoop.tracingChannel(); const states = new WeakMap(); @@ -137,9 +124,6 @@ export class CloudflareThinkPlugin extends BasePlugin { ? contextManager.wrapSpanForStore(runState.span) : currentSpanStore.getStore(); }); - this.unsubscribers.push(() => - channel.start?.unbindStore(currentSpanStore), - ); } const handlers: IsoChannelHandlers< @@ -166,7 +150,6 @@ export class CloudflareThinkPlugin extends BasePlugin { }; channel.subscribe(handlers); - this.unsubscribers.push(() => channel.unsubscribe(handlers)); } private subscribeToAISDKStreamTextSync(): void { @@ -186,7 +169,6 @@ export class CloudflareThinkPlugin extends BasePlugin { }; channel.subscribe(handlers); - this.unsubscribers.push(() => channel.unsubscribe(handlers)); } private subscribeToAISDKStreamTextAsync(): void { @@ -206,7 +188,6 @@ export class CloudflareThinkPlugin extends BasePlugin { }; channel.subscribe(handlers); - this.unsubscribers.push(() => channel.unsubscribe(handlers)); } private startAISDKStream(event: AISDKStreamEvent): void { @@ -382,3 +363,13 @@ function extractThinkTaskInput(input: unknown): unknown { } return input; } + +let cloudflareThinkInstrumentationConsumer: + | CloudflareThinkInstrumentationConsumer + | undefined; + +export function registerCloudflareThinkInstrumentation(): void { + cloudflareThinkInstrumentationConsumer ??= + new CloudflareThinkInstrumentationConsumer(); + cloudflareThinkInstrumentationConsumer.register(); +} diff --git a/js/src/instrumentation/plugins/cohere-channels.ts b/js/src/instrumentation/providers/cohere-channels.ts similarity index 100% rename from js/src/instrumentation/plugins/cohere-channels.ts rename to js/src/instrumentation/providers/cohere-channels.ts diff --git a/js/src/instrumentation/plugins/cohere-plugin.test.ts b/js/src/instrumentation/providers/cohere-instrumentation.test.ts similarity index 99% rename from js/src/instrumentation/plugins/cohere-plugin.test.ts rename to js/src/instrumentation/providers/cohere-instrumentation.test.ts index c2ea4fbf1..d7bd3603c 100644 --- a/js/src/instrumentation/plugins/cohere-plugin.test.ts +++ b/js/src/instrumentation/providers/cohere-instrumentation.test.ts @@ -3,7 +3,7 @@ import { aggregateCohereChatStreamChunks, extractCohereResponseMetadata, parseCohereMetricsFromUsage, -} from "./cohere-plugin"; +} from "./cohere-instrumentation"; describe("parseCohereMetricsFromUsage", () => { it("parses usage and meta token counters", () => { diff --git a/js/src/instrumentation/plugins/cohere-plugin.ts b/js/src/instrumentation/providers/cohere-instrumentation.ts similarity index 89% rename from js/src/instrumentation/plugins/cohere-plugin.ts rename to js/src/instrumentation/providers/cohere-instrumentation.ts index afaea628c..14e21179c 100644 --- a/js/src/instrumentation/plugins/cohere-plugin.ts +++ b/js/src/instrumentation/providers/cohere-instrumentation.ts @@ -1,8 +1,6 @@ -import { BasePlugin } from "../core"; import { traceAsyncChannel, traceStreamingChannel, - unsubscribeAll, } from "../core/channel-tracing"; import { SpanTypeAttribute, isObject } from "../../../util/index"; import { processInputAttachments } from "../../wrappers/attachment-utils"; @@ -16,79 +14,67 @@ import type { CohereUsageLike, } from "../../vendor-sdk-types/cohere"; -export class CoherePlugin extends BasePlugin { - protected onEnable(): void { +class CohereInstrumentationConsumer { + public register(): void { this.subscribeToCohereChannels(); } - protected onDisable(): void { - this.unsubscribers = unsubscribeAll(this.unsubscribers); - } - private subscribeToCohereChannels(): void { - this.unsubscribers.push( - traceStreamingChannel(cohereChannels.chat, { - name: "cohere.chat", - type: SpanTypeAttribute.LLM, - extractInput: extractChatInputWithMetadata, - extractOutput: (result) => extractCohereChatOutput(result), - extractMetadata: (result) => extractCohereResponseMetadata(result), - extractMetrics: (result, startTime) => { - const metrics = parseCohereMetricsFromUsage(result); - if (startTime) { - metrics.time_to_first_token = getCurrentUnixTimestamp() - startTime; - } - return metrics; - }, - }), - ); + traceStreamingChannel(cohereChannels.chat, { + name: "cohere.chat", + type: SpanTypeAttribute.LLM, + extractInput: extractChatInputWithMetadata, + extractOutput: (result) => extractCohereChatOutput(result), + extractMetadata: (result) => extractCohereResponseMetadata(result), + extractMetrics: (result, startTime) => { + const metrics = parseCohereMetricsFromUsage(result); + if (startTime) { + metrics.time_to_first_token = getCurrentUnixTimestamp() - startTime; + } + return metrics; + }, + }); - this.unsubscribers.push( - traceStreamingChannel(cohereChannels.chatStream, { - name: "cohere.chatStream", - type: SpanTypeAttribute.LLM, - extractInput: extractChatInputWithMetadata, - extractOutput: () => undefined, - extractMetadata: () => undefined, - extractMetrics: () => ({}), - aggregateChunks: aggregateCohereChatStreamChunks, - }), - ); + traceStreamingChannel(cohereChannels.chatStream, { + name: "cohere.chatStream", + type: SpanTypeAttribute.LLM, + extractInput: extractChatInputWithMetadata, + extractOutput: () => undefined, + extractMetadata: () => undefined, + extractMetrics: () => ({}), + aggregateChunks: aggregateCohereChatStreamChunks, + }); - this.unsubscribers.push( - traceAsyncChannel(cohereChannels.embed, { - name: "cohere.embed", - type: SpanTypeAttribute.LLM, - extractInput: extractEmbedInputWithMetadata, - extractOutput: extractCohereEmbeddingOutput, - extractMetadata: (result) => extractCohereResponseMetadata(result), - extractMetrics: (result) => parseCohereMetricsFromUsage(result), - }), - ); + traceAsyncChannel(cohereChannels.embed, { + name: "cohere.embed", + type: SpanTypeAttribute.LLM, + extractInput: extractEmbedInputWithMetadata, + extractOutput: extractCohereEmbeddingOutput, + extractMetadata: (result) => extractCohereResponseMetadata(result), + extractMetrics: (result) => parseCohereMetricsFromUsage(result), + }); - this.unsubscribers.push( - traceAsyncChannel(cohereChannels.rerank, { - name: "cohere.rerank", - type: SpanTypeAttribute.LLM, - extractInput: extractRerankInputWithMetadata, - extractOutput: (result) => { - if (!isObject(result) || !Array.isArray(result.results)) { - return undefined; - } - - return result.results.slice(0, 100).map((item) => ({ - index: isObject(item) ? item.index : undefined, - relevance_score: isObject(item) - ? ((typeof item.relevanceScore === "number" - ? item.relevanceScore - : item.relevance_score) ?? null) - : null, - })); - }, - extractMetadata: (result) => extractCohereResponseMetadata(result), - extractMetrics: (result) => parseCohereMetricsFromUsage(result), - }), - ); + traceAsyncChannel(cohereChannels.rerank, { + name: "cohere.rerank", + type: SpanTypeAttribute.LLM, + extractInput: extractRerankInputWithMetadata, + extractOutput: (result) => { + if (!isObject(result) || !Array.isArray(result.results)) { + return undefined; + } + + return result.results.slice(0, 100).map((item) => ({ + index: isObject(item) ? item.index : undefined, + relevance_score: isObject(item) + ? ((typeof item.relevanceScore === "number" + ? item.relevanceScore + : item.relevance_score) ?? null) + : null, + })); + }, + extractMetadata: (result) => extractCohereResponseMetadata(result), + extractMetrics: (result) => parseCohereMetricsFromUsage(result), + }); } } @@ -949,3 +935,10 @@ export function aggregateCohereChatStreamChunks( output, }; } + +let cohereInstrumentationConsumer: CohereInstrumentationConsumer | undefined; + +export function registerCohereInstrumentation(): void { + cohereInstrumentationConsumer ??= new CohereInstrumentationConsumer(); + cohereInstrumentationConsumer.register(); +} diff --git a/js/src/instrumentation/plugins/cursor-sdk-channels.ts b/js/src/instrumentation/providers/cursor-sdk-channels.ts similarity index 100% rename from js/src/instrumentation/plugins/cursor-sdk-channels.ts rename to js/src/instrumentation/providers/cursor-sdk-channels.ts diff --git a/js/src/instrumentation/plugins/cursor-sdk-plugin.test.ts b/js/src/instrumentation/providers/cursor-sdk-instrumentation.test.ts similarity index 94% rename from js/src/instrumentation/plugins/cursor-sdk-plugin.test.ts rename to js/src/instrumentation/providers/cursor-sdk-instrumentation.test.ts index b22052c8c..c30169c35 100644 --- a/js/src/instrumentation/plugins/cursor-sdk-plugin.test.ts +++ b/js/src/instrumentation/providers/cursor-sdk-instrumentation.test.ts @@ -15,11 +15,11 @@ vi.mock("../../logger", () => ({ })); import iso from "../../isomorph"; -import { CursorSDKPlugin } from "./cursor-sdk-plugin"; +import { registerCursorSDKInstrumentation } from "./cursor-sdk-instrumentation"; const mockNewTracingChannel = iso.newTracingChannel as ReturnType; -describe("CursorSDKPlugin", () => { +describe("registerCursorSDKInstrumentation", () => { let handlersByName: Map; let spans: Array<{ end: ReturnType; @@ -34,7 +34,6 @@ describe("CursorSDKPlugin", () => { mockNewTracingChannel.mockImplementation((name: string) => ({ subscribe: vi.fn((handlers) => handlersByName.set(name, handlers)), tracePromise: vi.fn((fn) => fn()), - unsubscribe: vi.fn(), })); mockStartSpan.mockImplementation((args: any) => { const span = { @@ -56,9 +55,7 @@ describe("CursorSDKPlugin", () => { }); it("subscribes to Cursor SDK channels", () => { - const plugin = new CursorSDKPlugin(); - - plugin.enable(); + registerCursorSDKInstrumentation(); expect(handlersByName.has("orchestrion:@cursor/sdk:Agent.create")).toBe( true, @@ -73,8 +70,7 @@ describe("CursorSDKPlugin", () => { }); it("patches agents returned by Agent.create and traces send/wait", async () => { - const plugin = new CursorSDKPlugin(); - plugin.enable(); + registerCursorSDKInstrumentation(); const createHandlers = handlersByName.get( "orchestrion:@cursor/sdk:Agent.create", @@ -131,8 +127,7 @@ describe("CursorSDKPlugin", () => { }); it("captures stream tool calls and usage", async () => { - const plugin = new CursorSDKPlugin(); - plugin.enable(); + registerCursorSDKInstrumentation(); const sendHandlers = handlersByName.get( "orchestrion:@cursor/sdk:agent.send", @@ -214,8 +209,7 @@ describe("CursorSDKPlugin", () => { }); it("traces Agent.prompt without a nested send span", () => { - const plugin = new CursorSDKPlugin(); - plugin.enable(); + registerCursorSDKInstrumentation(); const promptHandlers = handlersByName.get( "orchestrion:@cursor/sdk:Agent.prompt", diff --git a/js/src/instrumentation/plugins/cursor-sdk-plugin.ts b/js/src/instrumentation/providers/cursor-sdk-instrumentation.ts similarity index 98% rename from js/src/instrumentation/plugins/cursor-sdk-plugin.ts rename to js/src/instrumentation/providers/cursor-sdk-instrumentation.ts index f01db5987..41118be17 100644 --- a/js/src/instrumentation/plugins/cursor-sdk-plugin.ts +++ b/js/src/instrumentation/providers/cursor-sdk-instrumentation.ts @@ -1,4 +1,4 @@ -import { BasePlugin, toLoggedError } from "../core"; +import { toLoggedError } from "../core"; import type { ChannelMessage } from "../core/channel-definitions"; import type { IsoChannelHandlers } from "../../isomorph"; import { debugLogger } from "../../debug-logger"; @@ -64,23 +64,15 @@ type PromptState = { startTime: number; }; -export class CursorSDKPlugin extends BasePlugin { +class CursorSDKInstrumentationConsumer { private promptDepth = 0; - protected onEnable(): void { + public register(): void { this.subscribeToAgentFactories(); this.subscribeToPrompt(); this.subscribeToSend(); } - protected onDisable(): void { - for (const unsubscribe of this.unsubscribers) { - unsubscribe(); - } - this.unsubscribers = []; - this.promptDepth = 0; - } - private subscribeToAgentFactories(): void { this.subscribeToAgentFactory(cursorSDKChannels.create); this.subscribeToAgentFactory(cursorSDKChannels.resume); @@ -98,9 +90,6 @@ export class CursorSDKPlugin extends BasePlugin { }; tracingChannel.subscribe(handlers); - this.unsubscribers.push(() => { - tracingChannel.unsubscribe(handlers); - }); } private subscribeToPrompt(): void { @@ -171,9 +160,6 @@ export class CursorSDKPlugin extends BasePlugin { }; channel.subscribe(handlers); - this.unsubscribers.push(() => { - channel.unsubscribe(handlers); - }); } private subscribeToSend(): void { @@ -267,9 +253,6 @@ export class CursorSDKPlugin extends BasePlugin { }; channel.subscribe(handlers); - this.unsubscribers.push(() => { - channel.unsubscribe(handlers); - }); } } @@ -1185,3 +1168,12 @@ function cleanMetrics(metrics: Record): Record { } return cleaned; } + +let cursorSDKInstrumentationConsumer: + | CursorSDKInstrumentationConsumer + | undefined; + +export function registerCursorSDKInstrumentation(): void { + cursorSDKInstrumentationConsumer ??= new CursorSDKInstrumentationConsumer(); + cursorSDKInstrumentationConsumer.register(); +} diff --git a/js/src/instrumentation/plugins/eve-plugin.test.ts b/js/src/instrumentation/providers/eve-instrumentation.test.ts similarity index 99% rename from js/src/instrumentation/plugins/eve-plugin.test.ts rename to js/src/instrumentation/providers/eve-instrumentation.test.ts index 38e15bef0..9e1a6bc28 100644 --- a/js/src/instrumentation/plugins/eve-plugin.test.ts +++ b/js/src/instrumentation/providers/eve-instrumentation.test.ts @@ -11,7 +11,7 @@ import * as braintrustExports from "../../exports"; import { configureNode } from "../../node/config"; import { Attachment, _exportsForTestingOnly, initLogger } from "../../logger"; import * as instrumentationExports from "../index"; -import { braintrustEveInstrumentation } from "./eve-plugin"; +import { braintrustEveInstrumentation } from "./eve-instrumentation"; import type { EveInstrumentationAttemptScope, EveInstrumentationHandlerContext, @@ -60,7 +60,7 @@ describe("braintrustEveInstrumentation", () => { beforeEach(() => { backgroundLogger = _exportsForTestingOnly.useTestBackgroundLogger(); initLogger({ - projectName: "eve-plugin.test.ts", + projectName: "eve-instrumentation.test.ts", projectId: "test-project-id", }); }); @@ -107,7 +107,7 @@ describe("braintrustEveInstrumentation", () => { it("records a turn with LLM and action spans", async () => { const provider = braintrustEveInstrumentation({ metadata: { - scenario: "eve-plugin-unit", + scenario: "eve-instrumentation-unit", testRunId: "test-run-flat-tree", }, }); @@ -272,7 +272,7 @@ describe("braintrustEveInstrumentation", () => { input: [{ content: "Search then read", role: "user" }], metadata: { "eve.session_id": "session-root", - scenario: "eve-plugin-unit", + scenario: "eve-instrumentation-unit", testRunId: "test-run-flat-tree", }, output: "Here is the Eve instrumentation guide.", @@ -300,7 +300,7 @@ describe("braintrustEveInstrumentation", () => { "eve.session_id": "session-root", model: "qwen/qwen3-30b-a3b", provider: "openrouter", - scenario: "eve-plugin-unit", + scenario: "eve-instrumentation-unit", testRunId: "test-run-flat-tree", }, span_attributes: { @@ -345,7 +345,7 @@ describe("braintrustEveInstrumentation", () => { input: { query: "Eve instrumentation" }, metadata: { "eve.session_id": "session-root", - scenario: "eve-plugin-unit", + scenario: "eve-instrumentation-unit", testRunId: "test-run-flat-tree", }, output: { hits: ["eve.dev/docs"] }, diff --git a/js/src/instrumentation/plugins/eve-plugin.ts b/js/src/instrumentation/providers/eve-instrumentation.ts similarity index 100% rename from js/src/instrumentation/plugins/eve-plugin.ts rename to js/src/instrumentation/providers/eve-instrumentation.ts diff --git a/js/src/instrumentation/plugins/flue-plugin.test.ts b/js/src/instrumentation/providers/flue-instrumentation.test.ts similarity index 99% rename from js/src/instrumentation/plugins/flue-plugin.test.ts rename to js/src/instrumentation/providers/flue-instrumentation.test.ts index 04901f8cc..ae6fbec8b 100644 --- a/js/src/instrumentation/plugins/flue-plugin.test.ts +++ b/js/src/instrumentation/providers/flue-instrumentation.test.ts @@ -64,7 +64,7 @@ vi.mock("../../logger", () => ({ }, })); -import { braintrustFlueInstrumentation } from "./flue-plugin"; +import { braintrustFlueInstrumentation } from "./flue-instrumentation"; describe("Flue instrumentation", () => { let spans: Array<{ diff --git a/js/src/instrumentation/plugins/flue-plugin.ts b/js/src/instrumentation/providers/flue-instrumentation.ts similarity index 100% rename from js/src/instrumentation/plugins/flue-plugin.ts rename to js/src/instrumentation/providers/flue-instrumentation.ts diff --git a/js/src/instrumentation/plugins/genkit-channels.ts b/js/src/instrumentation/providers/genkit-channels.ts similarity index 100% rename from js/src/instrumentation/plugins/genkit-channels.ts rename to js/src/instrumentation/providers/genkit-channels.ts diff --git a/js/src/instrumentation/plugins/genkit-plugin.test.ts b/js/src/instrumentation/providers/genkit-instrumentation.test.ts similarity index 91% rename from js/src/instrumentation/plugins/genkit-plugin.test.ts rename to js/src/instrumentation/providers/genkit-instrumentation.test.ts index dcf41c0f7..11f172e5b 100644 --- a/js/src/instrumentation/plugins/genkit-plugin.test.ts +++ b/js/src/instrumentation/providers/genkit-instrumentation.test.ts @@ -1,6 +1,6 @@ import { afterEach, beforeAll, beforeEach, describe, expect, it } from "vitest"; import { _exportsForTestingOnly, initLogger } from "../../logger"; -import { GenkitPlugin } from "./genkit-plugin"; +import { registerGenkitInstrumentation } from "./genkit-instrumentation"; import { genkitChannels } from "./genkit-channels"; function singleQueueStream( @@ -35,9 +35,7 @@ async function collectAsync(stream: AsyncIterable): Promise { return chunks; } -describe("GenkitPlugin stream patching", () => { - const plugin = new GenkitPlugin(); - +describe("registerGenkitInstrumentation stream patching", () => { beforeAll(async () => { await _exportsForTestingOnly.simulateLoginForTests(); }); @@ -45,18 +43,17 @@ describe("GenkitPlugin stream patching", () => { beforeEach(() => { _exportsForTestingOnly.useTestBackgroundLogger(); initLogger({ - projectName: "genkit-plugin.test.ts", + projectName: "genkit-instrumentation.test.ts", projectId: "test-project-id", }); }); afterEach(() => { - plugin.disable(); _exportsForTestingOnly.clearTestBackgroundLogger(); }); it("does not consume generateStream chunks before user code reads them", async () => { - plugin.enable(); + registerGenkitInstrumentation(); const stream = singleQueueStream([{ text: "hello" }, { text: " world" }]); const result = genkitChannels.generateStream.traceSync( @@ -85,7 +82,7 @@ describe("GenkitPlugin stream patching", () => { }); it("does not consume action.stream chunks before user code reads them", async () => { - plugin.enable(); + registerGenkitInstrumentation(); const stream = singleQueueStream(["first", "second"]); const action = Object.assign(() => Promise.resolve(), { __action: { diff --git a/js/src/instrumentation/plugins/genkit-plugin.ts b/js/src/instrumentation/providers/genkit-instrumentation.ts similarity index 85% rename from js/src/instrumentation/plugins/genkit-plugin.ts rename to js/src/instrumentation/providers/genkit-instrumentation.ts index 90edb66e4..e485295ea 100644 --- a/js/src/instrumentation/plugins/genkit-plugin.ts +++ b/js/src/instrumentation/providers/genkit-instrumentation.ts @@ -1,8 +1,7 @@ -import { BasePlugin, toLoggedError } from "../core"; +import { toLoggedError } from "../core"; import { traceAsyncChannel, traceSyncStreamChannel, - unsubscribeAll, } from "../core/channel-tracing"; import { isAsyncIterable, patchStreamIfNeeded } from "../core/stream-patcher"; import type { ChannelMessage } from "../core/channel-definitions"; @@ -38,61 +37,49 @@ type SpanState = { startTime: number; }; -export class GenkitPlugin extends BasePlugin { - protected onEnable(): void { +class GenkitInstrumentationConsumer { + public register(): void { this.subscribeToGenkitChannels(); } - protected onDisable(): void { - this.unsubscribers = unsubscribeAll(this.unsubscribers); - } - private subscribeToGenkitChannels(): void { - this.unsubscribers.push( - traceAsyncChannel(genkitChannels.generate, { - name: "genkit.generate", - type: SpanTypeAttribute.LLM, - extractInput: ([input]) => extractGenerateInput(input), - extractOutput: extractGenerateOutput, - extractMetadata: (result, event) => - extractGenerateResponseMetadata(result, event?.arguments?.[0]), - extractMetrics: (result) => parseGenkitUsageMetrics(result?.usage), - }), - ); + traceAsyncChannel(genkitChannels.generate, { + name: "genkit.generate", + type: SpanTypeAttribute.LLM, + extractInput: ([input]) => extractGenerateInput(input), + extractOutput: extractGenerateOutput, + extractMetadata: (result, event) => + extractGenerateResponseMetadata(result, event?.arguments?.[0]), + extractMetrics: (result) => parseGenkitUsageMetrics(result?.usage), + }); - this.unsubscribers.push( - traceSyncStreamChannel(genkitChannels.generateStream, { - name: "genkit.generateStream", - type: SpanTypeAttribute.LLM, - extractInput: ([input]) => extractGenerateInput(input), - patchResult: ({ result, span, startTime }) => - patchGenerateStreamResult(result, span, startTime), - }), - ); + traceSyncStreamChannel(genkitChannels.generateStream, { + name: "genkit.generateStream", + type: SpanTypeAttribute.LLM, + extractInput: ([input]) => extractGenerateInput(input), + patchResult: ({ result, span, startTime }) => + patchGenerateStreamResult(result, span, startTime), + }); - this.unsubscribers.push( - traceAsyncChannel(genkitChannels.embed, { - name: "genkit.embed", - type: SpanTypeAttribute.FUNCTION, - extractInput: ([params]) => extractEmbedInput(params), - extractOutput: (result) => summarizeEmbeddingResult(result), - extractMetadata: (_result, event) => - extractEmbedMetadata(event?.arguments?.[0]), - extractMetrics: () => ({}), - }), - ); + traceAsyncChannel(genkitChannels.embed, { + name: "genkit.embed", + type: SpanTypeAttribute.FUNCTION, + extractInput: ([params]) => extractEmbedInput(params), + extractOutput: (result) => summarizeEmbeddingResult(result), + extractMetadata: (_result, event) => + extractEmbedMetadata(event?.arguments?.[0]), + extractMetrics: () => ({}), + }); - this.unsubscribers.push( - traceAsyncChannel(genkitChannels.embedMany, { - name: "genkit.embedMany", - type: SpanTypeAttribute.FUNCTION, - extractInput: ([params]) => extractEmbedManyInput(params), - extractOutput: summarizeEmbeddingResult, - extractMetadata: (_result, event) => - extractEmbedMetadata(event?.arguments?.[0]), - extractMetrics: () => ({}), - }), - ); + traceAsyncChannel(genkitChannels.embedMany, { + name: "genkit.embedMany", + type: SpanTypeAttribute.FUNCTION, + extractInput: ([params]) => extractEmbedManyInput(params), + extractOutput: summarizeEmbeddingResult, + extractMetadata: (_result, event) => + extractEmbedMetadata(event?.arguments?.[0]), + extractMetrics: () => ({}), + }); this.subscribeToActionRun(); this.subscribeToActionSpan(); @@ -105,10 +92,8 @@ export class GenkitPlugin extends BasePlugin { ChannelMessage >; const states = new WeakMap(); - const unbindCurrentSpanStore = bindActionCurrentSpanStoreToStart( - tracingChannel, - states, - (event) => startActionRunSpan(event), + bindActionCurrentSpanStoreToStart(tracingChannel, states, (event) => + startActionRunSpan(event), ); const handlers: IsoChannelHandlers< @@ -147,10 +132,6 @@ export class GenkitPlugin extends BasePlugin { }; tracingChannel.subscribe(handlers); - this.unsubscribers.push(() => { - unbindCurrentSpanStore?.(); - tracingChannel.unsubscribe(handlers); - }); } private subscribeToActionSpan(): void { @@ -159,10 +140,8 @@ export class GenkitPlugin extends BasePlugin { ChannelMessage >; const states = new WeakMap(); - const unbindCurrentSpanStore = bindActionCurrentSpanStoreToStart( - tracingChannel, - states, - (event) => startActionSpan(event), + bindActionCurrentSpanStoreToStart(tracingChannel, states, (event) => + startActionSpan(event), ); const handlers: IsoChannelHandlers< @@ -202,25 +181,19 @@ export class GenkitPlugin extends BasePlugin { }; tracingChannel.subscribe(handlers); - this.unsubscribers.push(() => { - unbindCurrentSpanStore?.(); - tracingChannel.unsubscribe(handlers); - }); } private subscribeToActionStream(): void { - this.unsubscribers.push( - traceSyncStreamChannel(genkitChannels.actionStream, { - name: "genkit.action.stream", - type: SpanTypeAttribute.TASK, - extractInput: ([input], event) => ({ - input, - metadata: actionMetadataForLog(extractActionMetadata(event.self)), - }), - patchResult: ({ result, span, startTime }) => - patchActionStreamResult(result, span, startTime), + traceSyncStreamChannel(genkitChannels.actionStream, { + name: "genkit.action.stream", + type: SpanTypeAttribute.TASK, + extractInput: ([input], event) => ({ + input, + metadata: actionMetadataForLog(extractActionMetadata(event.self)), }), - ); + patchResult: ({ result, span, startTime }) => + patchActionStreamResult(result, span, startTime), + }); } } @@ -307,7 +280,7 @@ function bindActionCurrentSpanStoreToStart< tracingChannel: IsoTracingChannel>, states: WeakMap, create: (event: ChannelMessage) => SpanState | undefined, -): (() => void) | undefined { +): void { const state = _internalGetGlobalState(); const contextManager = state?.contextManager; const startChannel = tracingChannel.start as @@ -316,7 +289,6 @@ function bindActionCurrentSpanStoreToStart< store: CurrentSpanStore, callback: (event: ChannelMessage) => unknown, ) => void; - unbindStore?: (store: CurrentSpanStore) => void; } & object) | undefined; const currentSpanStore = contextManager @@ -328,7 +300,7 @@ function bindActionCurrentSpanStoreToStart< : undefined; if (!startChannel?.bindStore || !currentSpanStore) { - return undefined; + return; } startChannel.bindStore(currentSpanStore, (event) => { @@ -339,10 +311,6 @@ function bindActionCurrentSpanStoreToStart< ? contextManager!.wrapSpanForStore(state.span) : currentSpanStore.getStore(); }); - - return () => { - startChannel.unbindStore?.(currentSpanStore); - }; } function normalizeInput(input: GenkitGenerateInput): GenkitGenerateInput { @@ -771,3 +739,10 @@ function pickNumberMetrics( function stringValue(value: unknown): string | undefined { return typeof value === "string" ? value : undefined; } + +let genkitInstrumentationConsumer: GenkitInstrumentationConsumer | undefined; + +export function registerGenkitInstrumentation(): void { + genkitInstrumentationConsumer ??= new GenkitInstrumentationConsumer(); + genkitInstrumentationConsumer.register(); +} diff --git a/js/src/instrumentation/plugins/github-copilot-channels.ts b/js/src/instrumentation/providers/github-copilot-channels.ts similarity index 100% rename from js/src/instrumentation/plugins/github-copilot-channels.ts rename to js/src/instrumentation/providers/github-copilot-channels.ts diff --git a/js/src/instrumentation/plugins/github-copilot-plugin.test.ts b/js/src/instrumentation/providers/github-copilot-instrumentation.test.ts similarity index 79% rename from js/src/instrumentation/plugins/github-copilot-plugin.test.ts rename to js/src/instrumentation/providers/github-copilot-instrumentation.test.ts index 17ecb38d8..5e88b3b51 100644 --- a/js/src/instrumentation/plugins/github-copilot-plugin.test.ts +++ b/js/src/instrumentation/providers/github-copilot-instrumentation.test.ts @@ -1,8 +1,8 @@ import { describe, expect, it } from "vitest"; import { - GitHubCopilotPlugin, + registerGitHubCopilotInstrumentation, extractMetricsFromUsage, -} from "./github-copilot-plugin"; +} from "./github-copilot-instrumentation"; describe("extractMetricsFromUsage", () => { it("maps input/output tokens to standard metric keys", () => { @@ -119,38 +119,8 @@ describe("extractMetricsFromUsage", () => { }); }); -describe("GitHubCopilotPlugin lifecycle", () => { - it("enables without throwing", () => { - const plugin = new GitHubCopilotPlugin(); - expect(() => plugin.enable()).not.toThrow(); - plugin.disable(); - }); - - it("disables without throwing", () => { - const plugin = new GitHubCopilotPlugin(); - plugin.enable(); - expect(() => plugin.disable()).not.toThrow(); - }); - - it("enable is idempotent", () => { - const plugin = new GitHubCopilotPlugin(); - plugin.enable(); - expect(() => plugin.enable()).not.toThrow(); - plugin.disable(); - }); - - it("disable is idempotent", () => { - const plugin = new GitHubCopilotPlugin(); - plugin.enable(); - plugin.disable(); - expect(() => plugin.disable()).not.toThrow(); - }); - - it("can be re-enabled after disable", () => { - const plugin = new GitHubCopilotPlugin(); - plugin.enable(); - plugin.disable(); - expect(() => plugin.enable()).not.toThrow(); - plugin.disable(); +describe("registerGitHubCopilotInstrumentation", () => { + it("registers without throwing", () => { + expect(() => registerGitHubCopilotInstrumentation()).not.toThrow(); }); }); diff --git a/js/src/instrumentation/plugins/github-copilot-plugin.ts b/js/src/instrumentation/providers/github-copilot-instrumentation.ts similarity index 97% rename from js/src/instrumentation/plugins/github-copilot-plugin.ts rename to js/src/instrumentation/providers/github-copilot-instrumentation.ts index 8fa156cfc..b5f14976f 100644 --- a/js/src/instrumentation/plugins/github-copilot-plugin.ts +++ b/js/src/instrumentation/providers/github-copilot-instrumentation.ts @@ -1,4 +1,3 @@ -import { BasePlugin } from "../core"; import type { IsoChannelHandlers } from "../../isomorph"; import { startSpan as startBaseSpan } from "../../logger"; import type { Span } from "../../logger"; @@ -672,7 +671,7 @@ function isGitHubCopilotSession(value: unknown): value is GitHubCopilotSession { } // --------------------------------------------------------------------------- -// Plugin and handler factory +// Consumer and handler factory // --------------------------------------------------------------------------- // eslint-disable-next-line @typescript-eslint/no-explicit-any @@ -756,18 +755,11 @@ function makeSessionHandlers( }; } -export class GitHubCopilotPlugin extends BasePlugin { - protected onEnable(): void { +class GitHubCopilotInstrumentationConsumer { + public register(): void { this.subscribeToSessionChannels(); } - protected onDisable(): void { - for (const unsubscribe of this.unsubscribers) { - unsubscribe(); - } - this.unsubscribers = []; - } - private subscribeToSessionChannels(): void { const createChannel = gitHubCopilotChannels.createSession.tracingChannel(); const resumeChannel = gitHubCopilotChannels.resumeSession.tracingChannel(); @@ -787,10 +779,15 @@ export class GitHubCopilotPlugin extends BasePlugin { createChannel.subscribe(createHandlers); resumeChannel.subscribe(resumeHandlers); - - this.unsubscribers.push( - () => createChannel.unsubscribe(createHandlers), - () => resumeChannel.unsubscribe(resumeHandlers), - ); } } + +let gitHubCopilotInstrumentationConsumer: + | GitHubCopilotInstrumentationConsumer + | undefined; + +export function registerGitHubCopilotInstrumentation(): void { + gitHubCopilotInstrumentationConsumer ??= + new GitHubCopilotInstrumentationConsumer(); + gitHubCopilotInstrumentationConsumer.register(); +} diff --git a/js/src/instrumentation/plugins/google-adk-channels.ts b/js/src/instrumentation/providers/google-adk-channels.ts similarity index 100% rename from js/src/instrumentation/plugins/google-adk-channels.ts rename to js/src/instrumentation/providers/google-adk-channels.ts diff --git a/js/src/instrumentation/plugins/google-adk-plugin.test.ts b/js/src/instrumentation/providers/google-adk-instrumentation.test.ts similarity index 91% rename from js/src/instrumentation/plugins/google-adk-plugin.test.ts rename to js/src/instrumentation/providers/google-adk-instrumentation.test.ts index 073af75f4..f4efcfc7d 100644 --- a/js/src/instrumentation/plugins/google-adk-plugin.test.ts +++ b/js/src/instrumentation/providers/google-adk-instrumentation.test.ts @@ -15,7 +15,7 @@ vi.mock("../../isomorph", () => ({ }, })); -import { GoogleADKPlugin } from "./google-adk-plugin"; +import { registerGoogleADKInstrumentation } from "./google-adk-instrumentation"; import iso from "../../isomorph"; const mockNewTracingChannel = iso.newTracingChannel as ReturnType; @@ -44,26 +44,19 @@ vi.mock("../../logger", () => ({ }, })); -describe("GoogleADKPlugin", () => { - let plugin: GoogleADKPlugin; +describe("registerGoogleADKInstrumentation", () => { let mockChannel: any; let subscribeSpy: any; - let unsubscribeSpy: any; let bindStoreSpy: any; - let unbindStoreSpy: any; beforeEach(() => { subscribeSpy = vi.fn(); - unsubscribeSpy = vi.fn(); bindStoreSpy = vi.fn(); - unbindStoreSpy = vi.fn(); mockChannel = { subscribe: subscribeSpy, - unsubscribe: unsubscribeSpy, hasSubscribers: false, start: { bindStore: bindStoreSpy, - unbindStore: unbindStoreSpy, }, }; @@ -71,16 +64,15 @@ describe("GoogleADKPlugin", () => { mockStartSpan.mockClear(); mockInternalGetGlobalState.mockReset(); mockInternalGetGlobalState.mockReturnValue(undefined); - plugin = new GoogleADKPlugin(); }); afterEach(() => { vi.clearAllMocks(); }); - describe("enable/disable lifecycle", () => { - it("should subscribe to channels when enabled", () => { - plugin.enable(); + describe("registration", () => { + it("subscribes to channels", () => { + registerGoogleADKInstrumentation(); // Should subscribe to 3 channels: runner.runAsync, agent.runAsync, tool.runAsync expect(mockNewTracingChannel).toHaveBeenCalledWith( @@ -94,43 +86,11 @@ describe("GoogleADKPlugin", () => { ); expect(subscribeSpy).toHaveBeenCalledTimes(3); }); - - it("should not subscribe multiple times if enabled twice", () => { - plugin.enable(); - const firstCallCount = subscribeSpy.mock.calls.length; - - plugin.enable(); - const secondCallCount = subscribeSpy.mock.calls.length; - - expect(firstCallCount).toBe(secondCallCount); - }); - - it("should unsubscribe from channels when disabled", () => { - plugin.enable(); - plugin.disable(); - - expect(unsubscribeSpy).toHaveBeenCalled(); - }); - - it("should clear unsubscribers array after disable", () => { - plugin.enable(); - plugin.disable(); - - // Enable again should re-subscribe - subscribeSpy.mockClear(); - plugin.enable(); - - expect(subscribeSpy).toHaveBeenCalledTimes(3); - }); - - it("should not crash when disabled without being enabled", () => { - expect(() => plugin.disable()).not.toThrow(); - }); }); describe("runner.runAsync channel", () => { it("should create a TASK span with runner metadata on start", () => { - plugin.enable(); + registerGoogleADKInstrumentation(); // Find the first subscribe call (runner channel) const handlers = subscribeSpy.mock.calls[0][0]; @@ -163,7 +123,7 @@ describe("GoogleADKPlugin", () => { }); it("should handle stream end with async iterable result", () => { - plugin.enable(); + registerGoogleADKInstrumentation(); const handlers = subscribeSpy.mock.calls[0][0]; const event: any = { @@ -196,7 +156,7 @@ describe("GoogleADKPlugin", () => { }); it("should handle error events", () => { - plugin.enable(); + registerGoogleADKInstrumentation(); const handlers = subscribeSpy.mock.calls[0][0]; const event: any = { @@ -228,7 +188,7 @@ describe("GoogleADKPlugin", () => { }, } as any); - plugin.enable(); + registerGoogleADKInstrumentation(); expect(bindStoreSpy).toHaveBeenNthCalledWith( 1, @@ -367,7 +327,7 @@ describe("GoogleADKPlugin", () => { ])( "normalizes $name", async ({ usageMetadata, expectedMetrics, absentMetrics }) => { - plugin.enable(); + registerGoogleADKInstrumentation(); const handlers = subscribeSpy.mock.calls[0][0]; const event: any = { @@ -398,7 +358,7 @@ describe("GoogleADKPlugin", () => { ); it("preserves explicitly reported zero usage", async () => { - plugin.enable(); + registerGoogleADKInstrumentation(); const handlers = subscribeSpy.mock.calls[0][0]; const event: any = { @@ -447,7 +407,7 @@ describe("GoogleADKPlugin", () => { describe("agent.runAsync channel", () => { it("should create a TASK span with agent metadata on start", () => { - plugin.enable(); + registerGoogleADKInstrumentation(); // Agent channel is the second subscribe call const handlers = subscribeSpy.mock.calls[1][0]; @@ -474,7 +434,7 @@ describe("GoogleADKPlugin", () => { }); it("uses the invoked agent instance for names when parent context still points at a parent agent", () => { - plugin.enable(); + registerGoogleADKInstrumentation(); const handlers = subscribeSpy.mock.calls[1][0]; const event = { @@ -511,7 +471,7 @@ describe("GoogleADKPlugin", () => { }); it("should handle agent without a name gracefully", () => { - plugin.enable(); + registerGoogleADKInstrumentation(); const handlers = subscribeSpy.mock.calls[1][0]; const event = { @@ -537,7 +497,7 @@ describe("GoogleADKPlugin", () => { }, } as any); - plugin.enable(); + registerGoogleADKInstrumentation(); expect(bindStoreSpy).toHaveBeenNthCalledWith( 2, @@ -588,7 +548,7 @@ describe("GoogleADKPlugin", () => { .mockImplementationOnce(() => runnerSpan) .mockImplementationOnce(() => agentSpan); - plugin.enable(); + registerGoogleADKInstrumentation(); const runnerHandlers = subscribeSpy.mock.calls[0][0]; const agentHandlers = subscribeSpy.mock.calls[1][0]; @@ -638,7 +598,7 @@ describe("GoogleADKPlugin", () => { describe("tool.runAsync channel", () => { it("should create a TOOL span on start", () => { - plugin.enable(); + registerGoogleADKInstrumentation(); // Tool channel is the third subscribe call const handlers = subscribeSpy.mock.calls[2][0]; @@ -675,7 +635,7 @@ describe("GoogleADKPlugin", () => { }); it("should log output and metrics on asyncEnd", () => { - plugin.enable(); + registerGoogleADKInstrumentation(); const handlers = subscribeSpy.mock.calls[2][0]; const event: any = { @@ -706,7 +666,7 @@ describe("GoogleADKPlugin", () => { }); it("should handle tool execution errors", () => { - plugin.enable(); + registerGoogleADKInstrumentation(); const handlers = subscribeSpy.mock.calls[2][0]; const event: any = { diff --git a/js/src/instrumentation/plugins/google-adk-plugin.ts b/js/src/instrumentation/providers/google-adk-instrumentation.ts similarity index 95% rename from js/src/instrumentation/plugins/google-adk-plugin.ts rename to js/src/instrumentation/providers/google-adk-instrumentation.ts index 623264016..994779305 100644 --- a/js/src/instrumentation/plugins/google-adk-plugin.ts +++ b/js/src/instrumentation/providers/google-adk-instrumentation.ts @@ -1,4 +1,3 @@ -import { BasePlugin } from "../core"; import type { ChannelMessage } from "../core/channel-definitions"; import { isAsyncIterable, patchStreamIfNeeded } from "../core/stream-patcher"; import type { IsoChannelHandlers, IsoTracingChannel } from "../../isomorph"; @@ -51,9 +50,9 @@ type GoogleADKStreamChannel = | typeof googleADKChannels.agentRunAsync; /** - * Auto-instrumentation plugin for the Google ADK. + * Internal auto-instrumentation consumer for the Google ADK. * - * This plugin subscribes to orchestrion channels for Google ADK methods + * This consumer subscribes to orchestrion channels for Google ADK methods * and creates Braintrust spans to track: * - Runner.runAsync — top-level agent execution (TASK span) * - BaseAgent.runAsync — individual agent invocations (TASK span) @@ -62,25 +61,16 @@ type GoogleADKStreamChannel = * LLM calls made through ADK are automatically captured by the existing * @google/genai instrumentation since ADK uses GenAI internally. */ -export class GoogleADKPlugin extends BasePlugin { +class GoogleADKInstrumentationConsumer { private activeRunnerSpans = new Map(); private activeAgentSpans = new Map(); - protected onEnable(): void { + public register(): void { this.subscribeToRunnerRunAsync(); this.subscribeToAgentRunAsync(); this.subscribeToToolRunAsync(); } - protected onDisable(): void { - for (const unsubscribe of this.unsubscribers) { - unsubscribe(); - } - this.unsubscribers = []; - this.activeRunnerSpans.clear(); - this.activeAgentSpans.clear(); - } - private subscribeToRunnerRunAsync(): void { const tracingChannel = googleADKChannels.runnerRunAsync.tracingChannel() as IsoTracingChannel< @@ -126,11 +116,7 @@ export class GoogleADKPlugin extends BasePlugin { return { span, startTime, events: [], contextKey }; }; - const unbindCurrentSpanStore = bindCurrentSpanStoreToStart( - tracingChannel, - states, - createState, - ); + bindCurrentSpanStoreToStart(tracingChannel, states, createState); const handlers: IsoChannelHandlers< ChannelMessage @@ -189,10 +175,6 @@ export class GoogleADKPlugin extends BasePlugin { }; tracingChannel.subscribe(handlers); - this.unsubscribers.push(() => { - unbindCurrentSpanStore?.(); - tracingChannel.unsubscribe(handlers); - }); } private subscribeToAgentRunAsync(): void { @@ -262,11 +244,7 @@ export class GoogleADKPlugin extends BasePlugin { return { span, startTime, events: [], contextKey, name: agentName }; }; - const unbindCurrentSpanStore = bindCurrentSpanStoreToStart( - tracingChannel, - states, - createState, - ); + bindCurrentSpanStoreToStart(tracingChannel, states, createState); const handlers: IsoChannelHandlers< ChannelMessage @@ -324,10 +302,6 @@ export class GoogleADKPlugin extends BasePlugin { }; tracingChannel.subscribe(handlers); - this.unsubscribers.push(() => { - unbindCurrentSpanStore?.(); - tracingChannel.unsubscribe(handlers); - }); } private subscribeToToolRunAsync(): void { @@ -413,9 +387,6 @@ export class GoogleADKPlugin extends BasePlugin { }; tracingChannel.subscribe(handlers); - this.unsubscribers.push(() => { - tracingChannel.unsubscribe(handlers); - }); } } @@ -536,7 +507,7 @@ function bindCurrentSpanStoreToStart< tracingChannel: IsoTracingChannel>, states: WeakMap, create: (event: ChannelMessage) => TState, -): (() => void) | undefined { +): void { const state = _internalGetGlobalState(); const contextManager = state?.contextManager; const startChannel = tracingChannel.start as @@ -545,7 +516,6 @@ function bindCurrentSpanStoreToStart< store: CurrentSpanStore, callback: (event: ChannelMessage) => unknown, ) => void; - unbindStore?: (store: CurrentSpanStore) => void; } & object) | undefined; const currentSpanStore = contextManager @@ -557,7 +527,7 @@ function bindCurrentSpanStoreToStart< : undefined; if (!startChannel?.bindStore || !currentSpanStore) { - return undefined; + return; } startChannel.bindStore(currentSpanStore, (event) => { @@ -566,10 +536,6 @@ function bindCurrentSpanStoreToStart< ).span; return contextManager.wrapSpanForStore(span); }); - - return () => { - startChannel.unbindStore?.(currentSpanStore); - }; } // ---- Helper functions ---- @@ -1026,3 +992,12 @@ function cleanMetrics(metrics: Record): Record { } return cleaned; } + +let googleADKInstrumentationConsumer: + | GoogleADKInstrumentationConsumer + | undefined; + +export function registerGoogleADKInstrumentation(): void { + googleADKInstrumentationConsumer ??= new GoogleADKInstrumentationConsumer(); + googleADKInstrumentationConsumer.register(); +} diff --git a/js/src/instrumentation/plugins/google-genai-channels.ts b/js/src/instrumentation/providers/google-genai-channels.ts similarity index 100% rename from js/src/instrumentation/plugins/google-genai-channels.ts rename to js/src/instrumentation/providers/google-genai-channels.ts diff --git a/js/src/instrumentation/plugins/google-genai-plugin.test.ts b/js/src/instrumentation/providers/google-genai-instrumentation.test.ts similarity index 95% rename from js/src/instrumentation/plugins/google-genai-plugin.test.ts rename to js/src/instrumentation/providers/google-genai-instrumentation.test.ts index f14690e73..175cfb309 100644 --- a/js/src/instrumentation/plugins/google-genai-plugin.test.ts +++ b/js/src/instrumentation/providers/google-genai-instrumentation.test.ts @@ -25,7 +25,7 @@ vi.mock("../../isomorph", () => ({ }, })); -import { GoogleGenAIPlugin } from "./google-genai-plugin"; +import { registerGoogleGenAIInstrumentation } from "./google-genai-instrumentation"; import { startSpan } from "../../logger"; import iso from "../../isomorph"; @@ -50,66 +50,27 @@ vi.mock("../../logger", () => ({ }, })); -describe("GoogleGenAIPlugin", () => { - let plugin: GoogleGenAIPlugin; +describe("registerGoogleGenAIInstrumentation", () => { let mockChannel: any; let subscribeSpy: any; - let unsubscribeSpy: any; beforeEach(() => { subscribeSpy = vi.fn(); - unsubscribeSpy = vi.fn(); mockChannel = { subscribe: subscribeSpy, - unsubscribe: unsubscribeSpy, hasSubscribers: false, }; mockNewTracingChannel.mockReturnValue(mockChannel); - plugin = new GoogleGenAIPlugin(); }); afterEach(() => { vi.clearAllMocks(); }); - describe("enable/disable lifecycle", () => { - it("should not subscribe multiple times if enabled twice", () => { - plugin.enable(); - const firstCallCount = subscribeSpy.mock.calls.length; - - plugin.enable(); - const secondCallCount = subscribeSpy.mock.calls.length; - - expect(firstCallCount).toBe(secondCallCount); - }); - - it("should unsubscribe from channels when disabled", () => { - plugin.enable(); - plugin.disable(); - - expect(unsubscribeSpy).toHaveBeenCalled(); - }); - - it("should clear unsubscribers array after disable", () => { - plugin.enable(); - plugin.disable(); - - // Enable again should re-subscribe - subscribeSpy.mockClear(); - plugin.enable(); - - expect(subscribeSpy).toHaveBeenCalled(); - }); - - it("should not crash when disabled without being enabled", () => { - expect(() => plugin.disable()).not.toThrow(); - }); - }); - describe("generateContent channel subscription", () => { it("should extract input correctly", () => { - plugin.enable(); + registerGoogleGenAIInstrumentation(); const subscribeCall = subscribeSpy.mock.calls.find( (call: any) => @@ -199,7 +160,7 @@ describe("GoogleGenAIPlugin", () => { ])( "normalizes $name", ({ usageMetadata, expectedMetrics, absentMetrics }) => { - plugin.enable(); + registerGoogleGenAIInstrumentation(); const handlers = subscribeSpy.mock.calls[0][0]; const event: any = { @@ -227,7 +188,7 @@ describe("GoogleGenAIPlugin", () => { ); it("preserves explicitly reported zero usage", () => { - plugin.enable(); + registerGoogleGenAIInstrumentation(); const handlers = subscribeSpy.mock.calls[0][0]; const event: any = { @@ -275,7 +236,7 @@ describe("GoogleGenAIPlugin", () => { describe("interactions.create channel subscription", () => { it("subscribes to the interactions.create channel", () => { - plugin.enable(); + registerGoogleGenAIInstrumentation(); expect(mockNewTracingChannel).toHaveBeenCalledWith( "orchestrion:@google/genai:interactions.create", @@ -284,7 +245,7 @@ describe("GoogleGenAIPlugin", () => { }); it("logs non-streaming interaction output and metrics", () => { - plugin.enable(); + registerGoogleGenAIInstrumentation(); const handlers = subscribeSpy.mock.calls[3][0]; const scheduledAt = new Date("2026-01-02T03:04:05.000Z"); @@ -412,7 +373,7 @@ describe("GoogleGenAIPlugin", () => { }); it("preserves zero and missing interaction usage values", () => { - plugin.enable(); + registerGoogleGenAIInstrumentation(); const handlers = subscribeSpy.mock.calls[3][0]; const event: any = { @@ -488,7 +449,7 @@ describe("GoogleGenAIPlugin", () => { }); it("does not trace background interaction tasks", () => { - plugin.enable(); + registerGoogleGenAIInstrumentation(); const handlers = subscribeSpy.mock.calls[3][0]; const event: any = { @@ -512,7 +473,7 @@ describe("GoogleGenAIPlugin", () => { }); it("aggregates streaming interaction events when consumed", async () => { - plugin.enable(); + registerGoogleGenAIInstrumentation(); async function* stream() { yield { @@ -599,7 +560,7 @@ describe("GoogleGenAIPlugin", () => { }); it("ends the interaction span when a stream errors", async () => { - plugin.enable(); + registerGoogleGenAIInstrumentation(); const streamError = new Error("stream failed"); async function* stream() { @@ -651,7 +612,7 @@ describe("Google GenAI serialization functions", () => { contents: "Hello world", }; - // Since these are private functions, we'll test them through the plugin behavior + // Since these are private functions, test them through consumer behavior. // For now, we'll verify the structure by examining what gets logged expect(params.model).toBe("gemini-pro"); expect(params.contents).toBe("Hello world"); diff --git a/js/src/instrumentation/plugins/google-genai-plugin.ts b/js/src/instrumentation/providers/google-genai-instrumentation.ts similarity index 92% rename from js/src/instrumentation/plugins/google-genai-plugin.ts rename to js/src/instrumentation/providers/google-genai-instrumentation.ts index fc70e112c..85e5ae049 100644 --- a/js/src/instrumentation/plugins/google-genai-plugin.ts +++ b/js/src/instrumentation/providers/google-genai-instrumentation.ts @@ -1,5 +1,4 @@ -import { BasePlugin } from "../core"; -import { traceStreamingChannel, unsubscribeAll } from "../core/channel-tracing"; +import { traceStreamingChannel } from "../core/channel-tracing"; import type { ChannelMessage, ErrorOf, @@ -74,29 +73,25 @@ function createWrapperParityEvent(args: { } /** - * Auto-instrumentation plugin for the Google GenAI SDK. + * Internal auto-instrumentation consumer for the Google GenAI SDK. * - * This plugin subscribes to orchestrion channels for Google GenAI SDK methods + * This consumer subscribes to orchestrion channels for Google GenAI SDK methods * and creates Braintrust spans to track: * - models.generateContent (non-streaming) * - models.generateContentStream (streaming) * - models.embedContent (embeddings) * - * The plugin handles: + * The consumer handles: * - Google-specific token metrics (promptTokenCount, candidatesTokenCount, cachedContentTokenCount) * - Processing streaming responses * - Converting inline data (images) to Attachment objects * - Tool calls (functionCall, functionResponse) and executable code results */ -export class GoogleGenAIPlugin extends BasePlugin { - protected onEnable(): void { +class GoogleGenAIInstrumentationConsumer { + public register(): void { this.subscribeToGoogleGenAIChannels(); } - protected onDisable(): void { - this.unsubscribers = unsubscribeAll(this.unsubscribers); - } - private subscribeToGoogleGenAIChannels(): void { this.subscribeToGenerateContentChannel(); this.subscribeToGenerateContentStreamChannel(); @@ -110,32 +105,28 @@ export class GoogleGenAIPlugin extends BasePlugin { ChannelMessage >; const states = new WeakMap(); - const unbindCurrentSpanStore = bindCurrentSpanStoreToStart( - tracingChannel, - states, - (event) => { - const params = event.arguments[0]; - const input = serializeGenerateContentInput(params); - const metadata = extractGenerateContentMetadata(params); - const span = startBaseSpan( - withSpanInstrumentationName( - { - name: "generate_content", - spanAttributes: { - type: SpanTypeAttribute.LLM, - }, - event: createWrapperParityEvent({ input, metadata }), + bindCurrentSpanStoreToStart(tracingChannel, states, (event) => { + const params = event.arguments[0]; + const input = serializeGenerateContentInput(params); + const metadata = extractGenerateContentMetadata(params); + const span = startBaseSpan( + withSpanInstrumentationName( + { + name: "generate_content", + spanAttributes: { + type: SpanTypeAttribute.LLM, }, - INSTRUMENTATION_NAMES.GOOGLE_GENAI, - ), - ); - - return { - span, - startTime: getCurrentUnixTimestamp(), - }; - }, - ); + event: createWrapperParityEvent({ input, metadata }), + }, + INSTRUMENTATION_NAMES.GOOGLE_GENAI, + ), + ); + + return { + span, + startTime: getCurrentUnixTimestamp(), + }; + }); const handlers: IsoChannelHandlers> = { @@ -192,10 +183,6 @@ export class GoogleGenAIPlugin extends BasePlugin { }; tracingChannel.subscribe(handlers); - this.unsubscribers.push(() => { - unbindCurrentSpanStore?.(); - tracingChannel.unsubscribe(handlers); - }); } private subscribeToGenerateContentStreamChannel(): void { @@ -228,9 +215,6 @@ export class GoogleGenAIPlugin extends BasePlugin { }; tracingChannel.subscribe(handlers); - this.unsubscribers.push(() => { - tracingChannel.unsubscribe(handlers); - }); } private subscribeToEmbedContentChannel(): void { @@ -239,32 +223,28 @@ export class GoogleGenAIPlugin extends BasePlugin { ChannelMessage >; const states = new WeakMap(); - const unbindCurrentSpanStore = bindCurrentSpanStoreToStart( - tracingChannel, - states, - (event) => { - const params = event.arguments[0]; - const input = serializeEmbedContentInput(params); - const metadata = extractEmbedContentMetadata(params); - const span = startBaseSpan( - withSpanInstrumentationName( - { - name: "embed_content", - spanAttributes: { - type: SpanTypeAttribute.LLM, - }, - event: createWrapperParityEvent({ input, metadata }), + bindCurrentSpanStoreToStart(tracingChannel, states, (event) => { + const params = event.arguments[0]; + const input = serializeEmbedContentInput(params); + const metadata = extractEmbedContentMetadata(params); + const span = startBaseSpan( + withSpanInstrumentationName( + { + name: "embed_content", + spanAttributes: { + type: SpanTypeAttribute.LLM, }, - INSTRUMENTATION_NAMES.GOOGLE_GENAI, - ), - ); - - return { - span, - startTime: getCurrentUnixTimestamp(), - }; - }, - ); + event: createWrapperParityEvent({ input, metadata }), + }, + INSTRUMENTATION_NAMES.GOOGLE_GENAI, + ), + ); + + return { + span, + startTime: getCurrentUnixTimestamp(), + }; + }); const handlers: IsoChannelHandlers> = { start: (event) => { @@ -316,33 +296,26 @@ export class GoogleGenAIPlugin extends BasePlugin { }; tracingChannel.subscribe(handlers); - this.unsubscribers.push(() => { - unbindCurrentSpanStore?.(); - tracingChannel.unsubscribe(handlers); - }); } private subscribeToInteractionsCreateChannel(): void { - this.unsubscribers.push( - traceStreamingChannel( - googleGenAIChannels.interactionsCreate as InteractionsCreateChannel, - { - name: "create_interaction", - shouldTrace: ([params]) => !isBackgroundInteractionCreate(params), - type: SpanTypeAttribute.LLM, - extractInput: ([params]) => ({ - input: serializeInteractionInput(params), - metadata: extractInteractionMetadata(params), - }), - extractOutput: (result) => serializeInteractionValue(result), - extractMetadata: (result) => - extractInteractionResponseMetadata(result), - extractMetrics: (result, startTime) => - cleanMetrics(extractInteractionMetrics(result, startTime)), - aggregateChunks: (chunks, _result, _event, startTime) => - aggregateInteractionEvents(chunks, startTime), - }, - ), + traceStreamingChannel( + googleGenAIChannels.interactionsCreate as InteractionsCreateChannel, + { + name: "create_interaction", + shouldTrace: ([params]) => !isBackgroundInteractionCreate(params), + type: SpanTypeAttribute.LLM, + extractInput: ([params]) => ({ + input: serializeInteractionInput(params), + metadata: extractInteractionMetadata(params), + }), + extractOutput: (result) => serializeInteractionValue(result), + extractMetadata: (result) => extractInteractionResponseMetadata(result), + extractMetrics: (result, startTime) => + cleanMetrics(extractInteractionMetrics(result, startTime)), + aggregateChunks: (chunks, _result, _event, startTime) => + aggregateInteractionEvents(chunks, startTime), + }, ); } } @@ -372,7 +345,7 @@ function bindCurrentSpanStoreToStart< tracingChannel: IsoTracingChannel>, states: WeakMap, create: (event: StartOf) => SpanState, -): (() => void) | undefined { +): void { const state = _internalGetGlobalState(); const contextManager = state?.contextManager; const startChannel = tracingChannel.start as @@ -381,7 +354,6 @@ function bindCurrentSpanStoreToStart< store: CurrentSpanStore, callback: (event: ChannelMessage) => unknown, ) => void; - unbindStore?: (store: CurrentSpanStore) => void; } & object) | undefined; const currentSpanStore = contextManager @@ -393,7 +365,7 @@ function bindCurrentSpanStoreToStart< : undefined; if (!startChannel?.bindStore || !currentSpanStore) { - return undefined; + return; } startChannel.bindStore(currentSpanStore, (event) => { @@ -402,10 +374,6 @@ function bindCurrentSpanStoreToStart< ).span; return contextManager!.wrapSpanForStore(span); }); - - return () => { - startChannel.unbindStore?.(currentSpanStore); - }; } function logErrorAndEndSpan( @@ -1626,3 +1594,13 @@ function tryToDict(obj: unknown): Record | null { return null; } + +let googleGenAIInstrumentationConsumer: + | GoogleGenAIInstrumentationConsumer + | undefined; + +export function registerGoogleGenAIInstrumentation(): void { + googleGenAIInstrumentationConsumer ??= + new GoogleGenAIInstrumentationConsumer(); + googleGenAIInstrumentationConsumer.register(); +} diff --git a/js/src/instrumentation/plugins/groq-channels.ts b/js/src/instrumentation/providers/groq-channels.ts similarity index 100% rename from js/src/instrumentation/plugins/groq-channels.ts rename to js/src/instrumentation/providers/groq-channels.ts diff --git a/js/src/instrumentation/plugins/groq-plugin.test.ts b/js/src/instrumentation/providers/groq-instrumentation.test.ts similarity index 98% rename from js/src/instrumentation/plugins/groq-plugin.test.ts rename to js/src/instrumentation/providers/groq-instrumentation.test.ts index a140deb80..3b10736d4 100644 --- a/js/src/instrumentation/plugins/groq-plugin.test.ts +++ b/js/src/instrumentation/providers/groq-instrumentation.test.ts @@ -2,7 +2,7 @@ import { describe, expect, it } from "vitest"; import { aggregateGroqChatCompletionChunks, parseGroqMetrics, -} from "./groq-plugin"; +} from "./groq-instrumentation"; describe("parseGroqMetrics", () => { it("merges OpenAI-compatible usage metrics with Groq cache metrics", () => { diff --git a/js/src/instrumentation/plugins/groq-plugin.ts b/js/src/instrumentation/providers/groq-instrumentation.ts similarity index 58% rename from js/src/instrumentation/plugins/groq-plugin.ts rename to js/src/instrumentation/providers/groq-instrumentation.ts index 10b6d2c0a..6f2b85d5f 100644 --- a/js/src/instrumentation/plugins/groq-plugin.ts +++ b/js/src/instrumentation/providers/groq-instrumentation.ts @@ -1,8 +1,6 @@ -import { BasePlugin } from "../core"; import { traceAsyncChannel, traceStreamingChannel, - unsubscribeAll, } from "../core/channel-tracing"; import { SpanTypeAttribute } from "../../../util/index"; import { processInputAttachments } from "../../wrappers/attachment-utils"; @@ -10,63 +8,53 @@ import { getCurrentUnixTimestamp } from "../../util"; import { aggregateChatCompletionChunks, parseMetricsFromUsage, -} from "./openai-plugin"; +} from "./openai-instrumentation"; import { groqChannels } from "./groq-channels"; import type { GroqChatCompletion, GroqChatCompletionChunk, } from "../../vendor-sdk-types/groq"; -export class GroqPlugin extends BasePlugin { - protected onEnable(): void { - this.unsubscribers.push( - traceStreamingChannel(groqChannels.chatCompletionsCreate, { - name: "groq.chat.completions.create", - type: SpanTypeAttribute.LLM, - extractInput: ([params]) => { - const { messages, ...metadata } = params; - return { - input: processInputAttachments(messages), - metadata: { ...metadata, provider: "groq" }, - }; - }, - extractOutput: (result) => result?.choices, - extractMetrics: (result, startTime) => { - const metrics = parseGroqMetrics(result); - if (startTime) { - metrics.time_to_first_token = getCurrentUnixTimestamp() - startTime; - } - return metrics; - }, - aggregateChunks: aggregateGroqChatCompletionChunks, - }), - ); +export function registerGroqInstrumentation(): void { + traceStreamingChannel(groqChannels.chatCompletionsCreate, { + name: "groq.chat.completions.create", + type: SpanTypeAttribute.LLM, + extractInput: ([params]) => { + const { messages, ...metadata } = params; + return { + input: processInputAttachments(messages), + metadata: { ...metadata, provider: "groq" }, + }; + }, + extractOutput: (result) => result?.choices, + extractMetrics: (result, startTime) => { + const metrics = parseGroqMetrics(result); + if (startTime) { + metrics.time_to_first_token = getCurrentUnixTimestamp() - startTime; + } + return metrics; + }, + aggregateChunks: aggregateGroqChatCompletionChunks, + }); - this.unsubscribers.push( - traceAsyncChannel(groqChannels.embeddingsCreate, { - name: "groq.embeddings.create", - type: SpanTypeAttribute.LLM, - extractInput: ([params]) => { - const { input, ...metadata } = params; - return { - input, - metadata: { ...metadata, provider: "groq" }, - }; - }, - extractOutput: (result) => { - const embedding = result?.data?.[0]?.embedding; - return Array.isArray(embedding) - ? { embedding_length: embedding.length } - : undefined; - }, - extractMetrics: (result) => parseGroqMetrics(result), - }), - ); - } - - protected onDisable(): void { - this.unsubscribers = unsubscribeAll(this.unsubscribers); - } + traceAsyncChannel(groqChannels.embeddingsCreate, { + name: "groq.embeddings.create", + type: SpanTypeAttribute.LLM, + extractInput: ([params]) => { + const { input, ...metadata } = params; + return { + input, + metadata: { ...metadata, provider: "groq" }, + }; + }, + extractOutput: (result) => { + const embedding = result?.data?.[0]?.embedding; + return Array.isArray(embedding) + ? { embedding_length: embedding.length } + : undefined; + }, + extractMetrics: (result) => parseGroqMetrics(result), + }); } export function parseGroqMetrics( diff --git a/js/src/instrumentation/plugins/harness-agent-context.test.ts b/js/src/instrumentation/providers/harness-agent-context.test.ts similarity index 100% rename from js/src/instrumentation/plugins/harness-agent-context.test.ts rename to js/src/instrumentation/providers/harness-agent-context.test.ts diff --git a/js/src/instrumentation/plugins/huggingface-channels.ts b/js/src/instrumentation/providers/huggingface-channels.ts similarity index 100% rename from js/src/instrumentation/plugins/huggingface-channels.ts rename to js/src/instrumentation/providers/huggingface-channels.ts diff --git a/js/src/instrumentation/plugins/huggingface-plugin.test.ts b/js/src/instrumentation/providers/huggingface-instrumentation.test.ts similarity index 98% rename from js/src/instrumentation/plugins/huggingface-plugin.test.ts rename to js/src/instrumentation/providers/huggingface-instrumentation.test.ts index 8b2b80286..6c55e8bbe 100644 --- a/js/src/instrumentation/plugins/huggingface-plugin.test.ts +++ b/js/src/instrumentation/providers/huggingface-instrumentation.test.ts @@ -2,7 +2,7 @@ import { describe, expect, it } from "vitest"; import { aggregateChatCompletionChunks, extractResponseMetadata, -} from "./huggingface-plugin"; +} from "./huggingface-instrumentation"; describe("extractResponseMetadata", () => { it("keeps allowlisted response fields", () => { diff --git a/js/src/instrumentation/plugins/huggingface-plugin.ts b/js/src/instrumentation/providers/huggingface-instrumentation.ts similarity index 87% rename from js/src/instrumentation/plugins/huggingface-plugin.ts rename to js/src/instrumentation/providers/huggingface-instrumentation.ts index 797c3bf82..0407adffa 100644 --- a/js/src/instrumentation/plugins/huggingface-plugin.ts +++ b/js/src/instrumentation/providers/huggingface-instrumentation.ts @@ -1,10 +1,8 @@ import { traceAsyncChannel, traceSyncStreamChannel, - unsubscribeAll, } from "../core/channel-tracing"; import { isAsyncIterable, patchStreamIfNeeded } from "../core/stream-patcher"; -import { BasePlugin } from "../core"; import { SpanTypeAttribute, isObject } from "../../../util/index"; import { getCurrentUnixTimestamp } from "../../util"; import { parseMetricsFromUsage } from "../../openai-utils"; @@ -41,62 +39,54 @@ const RESPONSE_METADATA_ALLOWLIST = new Set([ "object", ]); -export class HuggingFacePlugin extends BasePlugin { - protected onEnable(): void { - this.unsubscribers.push( - traceAsyncChannel(huggingFaceChannels.chatCompletion, { - name: "huggingface.chat_completion", - type: SpanTypeAttribute.LLM, - extractInput: extractChatInputWithMetadata, - extractOutput: (result) => result?.choices, - extractMetadata: (result) => extractResponseMetadata(result), - extractMetrics: (result) => parseMetricsFromUsage(result?.usage), - }), - traceSyncStreamChannel(huggingFaceChannels.chatCompletionStream, { - name: "huggingface.chat_completion_stream", - type: SpanTypeAttribute.LLM, - extractInput: extractChatInputWithMetadata, - patchResult: ({ result, span, startTime }) => - patchChatCompletionStream({ - result, - span, - startTime, - }), - }), - traceAsyncChannel(huggingFaceChannels.textGeneration, { - name: "huggingface.text_generation", - type: SpanTypeAttribute.LLM, - extractInput: extractTextGenerationInputWithMetadata, - extractOutput: (result) => - isObject(result) ? { generated_text: result.generated_text } : result, - extractMetadata: extractTextGenerationMetadata, - extractMetrics: (result) => - extractTextGenerationMetrics(result?.details ?? null), - }), - traceSyncStreamChannel(huggingFaceChannels.textGenerationStream, { - name: "huggingface.text_generation_stream", - type: SpanTypeAttribute.LLM, - extractInput: extractTextGenerationInputWithMetadata, - patchResult: ({ result, span, startTime }) => - patchTextGenerationStream({ - result, - span, - startTime, - }), +export function registerHuggingFaceInstrumentation(): void { + traceAsyncChannel(huggingFaceChannels.chatCompletion, { + name: "huggingface.chat_completion", + type: SpanTypeAttribute.LLM, + extractInput: extractChatInputWithMetadata, + extractOutput: (result) => result?.choices, + extractMetadata: (result) => extractResponseMetadata(result), + extractMetrics: (result) => parseMetricsFromUsage(result?.usage), + }); + traceSyncStreamChannel(huggingFaceChannels.chatCompletionStream, { + name: "huggingface.chat_completion_stream", + type: SpanTypeAttribute.LLM, + extractInput: extractChatInputWithMetadata, + patchResult: ({ result, span, startTime }) => + patchChatCompletionStream({ + result, + span, + startTime, }), - traceAsyncChannel(huggingFaceChannels.featureExtraction, { - name: "huggingface.feature_extraction", - type: SpanTypeAttribute.LLM, - extractInput: extractFeatureExtractionInputWithMetadata, - extractOutput: summarizeFeatureExtractionOutput, - extractMetrics: () => ({}), + }); + traceAsyncChannel(huggingFaceChannels.textGeneration, { + name: "huggingface.text_generation", + type: SpanTypeAttribute.LLM, + extractInput: extractTextGenerationInputWithMetadata, + extractOutput: (result) => + isObject(result) ? { generated_text: result.generated_text } : result, + extractMetadata: extractTextGenerationMetadata, + extractMetrics: (result) => + extractTextGenerationMetrics(result?.details ?? null), + }); + traceSyncStreamChannel(huggingFaceChannels.textGenerationStream, { + name: "huggingface.text_generation_stream", + type: SpanTypeAttribute.LLM, + extractInput: extractTextGenerationInputWithMetadata, + patchResult: ({ result, span, startTime }) => + patchTextGenerationStream({ + result, + span, + startTime, }), - ); - } - - protected onDisable(): void { - this.unsubscribers = unsubscribeAll(this.unsubscribers); - } + }); + traceAsyncChannel(huggingFaceChannels.featureExtraction, { + name: "huggingface.feature_extraction", + type: SpanTypeAttribute.LLM, + extractInput: extractFeatureExtractionInputWithMetadata, + extractOutput: summarizeFeatureExtractionOutput, + extractMetrics: () => ({}), + }); } function addProviderMetadata( diff --git a/js/src/instrumentation/plugins/huggingface-transformers-channels.ts b/js/src/instrumentation/providers/huggingface-transformers-channels.ts similarity index 100% rename from js/src/instrumentation/plugins/huggingface-transformers-channels.ts rename to js/src/instrumentation/providers/huggingface-transformers-channels.ts diff --git a/js/src/instrumentation/plugins/huggingface-transformers-plugin.test.ts b/js/src/instrumentation/providers/huggingface-transformers-instrumentation.test.ts similarity index 97% rename from js/src/instrumentation/plugins/huggingface-transformers-plugin.test.ts rename to js/src/instrumentation/providers/huggingface-transformers-instrumentation.test.ts index 667fc19c7..b32e93579 100644 --- a/js/src/instrumentation/plugins/huggingface-transformers-plugin.test.ts +++ b/js/src/instrumentation/providers/huggingface-transformers-instrumentation.test.ts @@ -3,13 +3,13 @@ import { registerHuggingFaceTransformersPipeline, type HuggingFaceTransformersEventContext, } from "./huggingface-transformers-channels"; -import { _exportsForTestingOnly } from "./huggingface-transformers-plugin"; +import { _exportsForTestingOnly } from "./huggingface-transformers-instrumentation"; import type { HuggingFaceTransformersPipeline } from "../../vendor-sdk-types/huggingface-transformers"; const { extractInput, extractMetadata, extractOutput, isSupportedTask } = _exportsForTestingOnly; -describe("HuggingFaceTransformersPlugin extraction", () => { +describe("registerHuggingFaceTransformersInstrumentation extraction", () => { it("normalizes generation and chat payloads", () => { expect(extractInput("text-generation", ["Hello"])).toEqual([ { role: "user", content: "Hello" }, diff --git a/js/src/instrumentation/plugins/huggingface-transformers-plugin.ts b/js/src/instrumentation/providers/huggingface-transformers-instrumentation.ts similarity index 82% rename from js/src/instrumentation/plugins/huggingface-transformers-plugin.ts rename to js/src/instrumentation/providers/huggingface-transformers-instrumentation.ts index 8d41b535a..09e817685 100644 --- a/js/src/instrumentation/plugins/huggingface-transformers-plugin.ts +++ b/js/src/instrumentation/providers/huggingface-transformers-instrumentation.ts @@ -1,5 +1,4 @@ -import { BasePlugin } from "../core"; -import { traceAsyncChannel, unsubscribeAll } from "../core/channel-tracing"; +import { traceAsyncChannel } from "../core/channel-tracing"; import type { ChannelMessage } from "../core/channel-definitions"; import type { IsoChannelHandlers, IsoTracingChannel } from "../../isomorph"; import { SpanTypeAttribute, isObject } from "../../../util"; @@ -19,43 +18,37 @@ const REQUEST_METADATA_KEYS = [ "stop", ] as const; -export class HuggingFaceTransformersPlugin extends BasePlugin { - protected onEnable(): void { +class HuggingFaceTransformersInstrumentationConsumer { + public register(): void { this.subscribeToPipelineFactory(); - this.unsubscribers.push( - traceAsyncChannel(huggingFaceTransformersChannels.pipelineCall, { - name: (_args, event) => { - const task = getTask(event as HuggingFaceTransformersEventContext); - const operation = task?.replaceAll("-", "_") ?? "unknown"; - return `huggingface.transformers.${operation}`; - }, - type: SpanTypeAttribute.LLM, - shouldTrace: (_args, event) => - isSupportedHuggingFaceTransformersTask( - getTask(event as HuggingFaceTransformersEventContext), - ), - extractInput: (args, event) => ({ - input: extractInput( - getTask(event as HuggingFaceTransformersEventContext), - args, - ), - metadata: extractMetadata( - event as HuggingFaceTransformersEventContext, - args, - ), - }), - extractOutput: (result, event) => - extractOutput( - getTask(event as HuggingFaceTransformersEventContext), - result, - ), - extractMetrics: () => ({}), + traceAsyncChannel(huggingFaceTransformersChannels.pipelineCall, { + name: (_args, event) => { + const task = getTask(event as HuggingFaceTransformersEventContext); + const operation = task?.replaceAll("-", "_") ?? "unknown"; + return `huggingface.transformers.${operation}`; + }, + type: SpanTypeAttribute.LLM, + shouldTrace: (_args, event) => + isSupportedHuggingFaceTransformersTask( + getTask(event as HuggingFaceTransformersEventContext), + ), + extractInput: (args, event) => ({ + input: extractInput( + getTask(event as HuggingFaceTransformersEventContext), + args, + ), + metadata: extractMetadata( + event as HuggingFaceTransformersEventContext, + args, + ), }), - ); - } - - protected onDisable(): void { - this.unsubscribers = unsubscribeAll(this.unsubscribers); + extractOutput: (result, event) => + extractOutput( + getTask(event as HuggingFaceTransformersEventContext), + result, + ), + extractMetrics: () => ({}), + }); } private subscribeToPipelineFactory(): void { @@ -79,7 +72,6 @@ export class HuggingFaceTransformersPlugin extends BasePlugin { }; channel.subscribe(handlers); - this.unsubscribers.push(() => channel.unsubscribe(handlers)); } } @@ -321,3 +313,13 @@ export const _exportsForTestingOnly = { extractOutput, isSupportedTask: isSupportedHuggingFaceTransformersTask, }; + +let huggingFaceTransformersInstrumentationConsumer: + | HuggingFaceTransformersInstrumentationConsumer + | undefined; + +export function registerHuggingFaceTransformersInstrumentation(): void { + huggingFaceTransformersInstrumentationConsumer ??= + new HuggingFaceTransformersInstrumentationConsumer(); + huggingFaceTransformersInstrumentationConsumer.register(); +} diff --git a/js/src/instrumentation/plugins/instrumentation-names.test.ts b/js/src/instrumentation/providers/instrumentation-names.test.ts similarity index 100% rename from js/src/instrumentation/plugins/instrumentation-names.test.ts rename to js/src/instrumentation/providers/instrumentation-names.test.ts diff --git a/js/src/instrumentation/plugins/langchain-channels.ts b/js/src/instrumentation/providers/langchain-channels.ts similarity index 100% rename from js/src/instrumentation/plugins/langchain-channels.ts rename to js/src/instrumentation/providers/langchain-channels.ts diff --git a/js/src/instrumentation/plugins/langchain-plugin.test.ts b/js/src/instrumentation/providers/langchain-instrumentation.test.ts similarity index 79% rename from js/src/instrumentation/plugins/langchain-plugin.test.ts rename to js/src/instrumentation/providers/langchain-instrumentation.test.ts index 29ace20ab..570972a9d 100644 --- a/js/src/instrumentation/plugins/langchain-plugin.test.ts +++ b/js/src/instrumentation/providers/langchain-instrumentation.test.ts @@ -1,5 +1,5 @@ -import { describe, expect, it } from "vitest"; -import { LangChainPlugin } from "./langchain-plugin"; +import { beforeAll, describe, expect, it } from "vitest"; +import { registerLangChainInstrumentation } from "./langchain-instrumentation"; import { langChainChannels } from "./langchain-channels"; function createManager(handlers: unknown[] = []) { @@ -35,14 +35,15 @@ function createArgumentsObject(...args: unknown[]): IArguments { })(...args); } -describe("LangChainPlugin", () => { +describe("registerLangChainInstrumentation", () => { + beforeAll(() => { + registerLangChainInstrumentation(); + }); + it("injects a Braintrust callback handler into empty CallbackManager.configure() arguments", () => { - const plugin = new LangChainPlugin(); const args: unknown[] = []; - plugin.enable(); traceConfigureArguments(args); - plugin.disable(); expect(args[0]).toEqual([ expect.objectContaining({ @@ -52,12 +53,9 @@ describe("LangChainPlugin", () => { }); it("injects a Braintrust callback handler into real arguments objects", () => { - const plugin = new LangChainPlugin(); const args = createArgumentsObject(); - plugin.enable(); traceConfigureArgumentsObject(args); - plugin.disable(); expect(args[0]).toEqual([ expect.objectContaining({ @@ -67,12 +65,9 @@ describe("LangChainPlugin", () => { }); it("injects a Braintrust callback handler into CallbackManager.configure() results", () => { - const plugin = new LangChainPlugin(); const manager = createManager(); - plugin.enable(); traceConfigureResult(manager); - plugin.disable(); expect(manager.handlers).toHaveLength(1); expect(manager.handlers[0]).toMatchObject({ @@ -81,37 +76,25 @@ describe("LangChainPlugin", () => { }); it("does not inject duplicate handlers into the same manager", () => { - const plugin = new LangChainPlugin(); const manager = createManager(); - plugin.enable(); traceConfigureResult(manager); traceConfigureResult(manager); - plugin.disable(); expect(manager.handlers).toHaveLength(1); }); it("does not inject when a Braintrust callback handler is already present", () => { - const plugin = new LangChainPlugin(); const existingHandler = { name: "BraintrustCallbackHandler" }; const manager = createManager([existingHandler]); - plugin.enable(); traceConfigureResult(manager); - plugin.disable(); expect(manager.handlers).toEqual([existingHandler]); }); it("gracefully ignores undefined and non-manager results", () => { - const plugin = new LangChainPlugin(); - - plugin.enable(); - expect(() => traceConfigureResult(undefined)).not.toThrow(); expect(() => traceConfigureResult({ handlers: [] })).not.toThrow(); - - plugin.disable(); }); }); diff --git a/js/src/instrumentation/plugins/langchain-plugin.ts b/js/src/instrumentation/providers/langchain-instrumentation.ts similarity index 89% rename from js/src/instrumentation/plugins/langchain-plugin.ts rename to js/src/instrumentation/providers/langchain-instrumentation.ts index 85ef66083..42a5f7fd5 100644 --- a/js/src/instrumentation/plugins/langchain-plugin.ts +++ b/js/src/instrumentation/providers/langchain-instrumentation.ts @@ -1,4 +1,3 @@ -import { BasePlugin } from "../core"; import type { ChannelMessage } from "../core/channel-definitions"; import type { IsoChannelHandlers, IsoTracingChannel } from "../../isomorph"; import type { LangChainCallbackManager } from "../../vendor-sdk-types/langchain"; @@ -12,22 +11,14 @@ type LangChainConfigureChannel = | typeof langChainChannels.configure | typeof langChainChannels.configureSync; -export class LangChainPlugin extends BasePlugin { +class LangChainInstrumentationConsumer { private injectedManagers = new WeakSet(); - protected onEnable(): void { + public register(): void { this.subscribeToConfigure(langChainChannels.configure); this.subscribeToConfigure(langChainChannels.configureSync); } - protected onDisable(): void { - for (const unsubscribe of this.unsubscribers) { - unsubscribe(); - } - this.unsubscribers = []; - this.injectedManagers = new WeakSet(); - } - private subscribeToConfigure(channel: LangChainConfigureChannel): void { const tracingChannel: IsoTracingChannel< ChannelMessage @@ -45,9 +36,6 @@ export class LangChainPlugin extends BasePlugin { }; tracingChannel.subscribe(handlers); - this.unsubscribers.push(() => { - tracingChannel.unsubscribe(handlers); - }); } private injectHandler(result: unknown): void { @@ -126,3 +114,12 @@ function isBraintrustHandler(handler: unknown): boolean { Reflect.get(handler, "name") === BRAINTRUST_LANGCHAIN_CALLBACK_HANDLER_NAME ); } + +let langChainInstrumentationConsumer: + | LangChainInstrumentationConsumer + | undefined; + +export function registerLangChainInstrumentation(): void { + langChainInstrumentationConsumer ??= new LangChainInstrumentationConsumer(); + langChainInstrumentationConsumer.register(); +} diff --git a/js/src/instrumentation/plugins/langsmith-channels.ts b/js/src/instrumentation/providers/langsmith-channels.ts similarity index 100% rename from js/src/instrumentation/plugins/langsmith-channels.ts rename to js/src/instrumentation/providers/langsmith-channels.ts diff --git a/js/src/instrumentation/plugins/langsmith-plugin.test.ts b/js/src/instrumentation/providers/langsmith-instrumentation.test.ts similarity index 88% rename from js/src/instrumentation/plugins/langsmith-plugin.test.ts rename to js/src/instrumentation/providers/langsmith-instrumentation.test.ts index 8e9b3aed4..55a986a74 100644 --- a/js/src/instrumentation/plugins/langsmith-plugin.test.ts +++ b/js/src/instrumentation/providers/langsmith-instrumentation.test.ts @@ -1,37 +1,36 @@ import { afterEach, beforeAll, beforeEach, describe, expect, it } from "vitest"; import { configureNode } from "../../node/config"; import { _exportsForTestingOnly, initLogger } from "../../logger"; +import { configureInstrumentation } from "../registry"; import { langSmithChannels } from "./langsmith-channels"; -import { LangSmithPlugin } from "./langsmith-plugin"; +import { registerLangSmithInstrumentation } from "./langsmith-instrumentation"; try { + configureInstrumentation({ integrations: { langsmith: false } }); configureNode(); } catch { // Best-effort initialization for test environments. } -describe("LangSmithPlugin", () => { +describe("registerLangSmithInstrumentation", () => { let backgroundLogger: ReturnType< typeof _exportsForTestingOnly.useTestBackgroundLogger >; - let plugin: LangSmithPlugin; beforeAll(async () => { await _exportsForTestingOnly.simulateLoginForTests(); + registerLangSmithInstrumentation(); }); beforeEach(() => { backgroundLogger = _exportsForTestingOnly.useTestBackgroundLogger(); initLogger({ - projectName: "langsmith-plugin.test.ts", + projectName: "langsmith-instrumentation.test.ts", projectId: "test-project-id", }); - plugin = new LangSmithPlugin(); - plugin.enable(); }); afterEach(() => { - plugin.disable(); _exportsForTestingOnly.clearTestBackgroundLogger(); }); @@ -269,7 +268,7 @@ describe("LangSmithPlugin", () => { expect(await backgroundLogger.drain()).toEqual([]); }); - it("suppresses LangChain-serialized runs only when configured", async () => { + it("suppresses LangChain-serialized runs by default", async () => { const run = { id: "44444444-4444-4444-8444-444444444444", trace_id: "44444444-4444-4444-8444-444444444444", @@ -282,31 +281,5 @@ describe("LangSmithPlugin", () => { arguments: [run.id, run], }); expect(await backgroundLogger.drain()).toEqual([]); - - plugin.disable(); - plugin = new LangSmithPlugin({ skipLangChainRuns: false }); - plugin.enable(); - await langSmithChannels.updateRun.tracePromise(async () => undefined, { - arguments: [run.id, run], - }); - expect(await backgroundLogger.drain()).toHaveLength(1); - }); - - it("bounds the completed-run deduplication cache", () => { - const completedRuns = ( - plugin as unknown as { - completedRuns: { - get(key: string): true | undefined; - set(key: string, value: true): void; - }; - } - ).completedRuns; - - for (let index = 0; index <= 10_000; index++) { - completedRuns.set(`run-${index}`, true); - } - - expect(completedRuns.get("run-0")).toBeUndefined(); - expect(completedRuns.get("run-10000")).toBe(true); }); }); diff --git a/js/src/instrumentation/plugins/langsmith-plugin.ts b/js/src/instrumentation/providers/langsmith-instrumentation.ts similarity index 95% rename from js/src/instrumentation/plugins/langsmith-plugin.ts rename to js/src/instrumentation/providers/langsmith-instrumentation.ts index 5027fa694..1716b88cb 100644 --- a/js/src/instrumentation/plugins/langsmith-plugin.ts +++ b/js/src/instrumentation/providers/langsmith-instrumentation.ts @@ -11,8 +11,6 @@ import type { LangSmithBatchIngestRuns, LangSmithRun, } from "../../vendor-sdk-types/langsmith"; -import { BasePlugin } from "../core"; -import { unsubscribeAll } from "../core/channel-tracing"; import type { ChannelMessage } from "../core/channel-definitions"; import { langSmithChannels } from "./langsmith-channels"; @@ -21,7 +19,7 @@ type ActiveRun = { span: Span; }; -type LangSmithPluginOptions = { +type LangSmithInstrumentationOptions = { skipLangChainRuns?: boolean; }; @@ -43,19 +41,18 @@ const LLM_SETTING_KEYS = [ "response_format", ] as const; -export class LangSmithPlugin extends BasePlugin { +class LangSmithInstrumentationConsumer { private readonly activeRuns = new Map(); private readonly completedRuns = new LRUCache({ max: MAX_COMPLETED_RUNS, }); private readonly skipLangChainRuns: boolean; - constructor(options: LangSmithPluginOptions = {}) { - super(); + constructor(options: LangSmithInstrumentationOptions = {}) { this.skipLangChainRuns = options.skipLangChainRuns ?? true; } - protected onEnable(): void { + public register(): void { const createChannel = langSmithChannels.createRun.tracingChannel(); const createHandlers = { start: ( @@ -67,7 +64,6 @@ export class LangSmithPlugin extends BasePlugin { }, }; createChannel.subscribe(createHandlers); - this.unsubscribers.push(() => createChannel.unsubscribe(createHandlers)); const updateChannel = langSmithChannels.updateRun.tracingChannel(); const updateHandlers = { @@ -80,7 +76,6 @@ export class LangSmithPlugin extends BasePlugin { }, }; updateChannel.subscribe(updateHandlers); - this.unsubscribers.push(() => updateChannel.unsubscribe(updateHandlers)); const batchChannel = langSmithChannels.batchIngestRuns.tracingChannel(); const batchHandlers = { @@ -93,16 +88,6 @@ export class LangSmithPlugin extends BasePlugin { }, }; batchChannel.subscribe(batchHandlers); - this.unsubscribers.push(() => batchChannel.unsubscribe(batchHandlers)); - } - - protected onDisable(): void { - this.unsubscribers = unsubscribeAll(this.unsubscribers); - for (const { span } of this.activeRuns.values()) { - span.end(); - } - this.activeRuns.clear(); - this.completedRuns.clear(); } private processBatch(batch: LangSmithBatchIngestRuns): void { @@ -337,6 +322,19 @@ function mergeRuns( return Object.fromEntries(entries); } +let langSmithInstrumentationConsumer: + | LangSmithInstrumentationConsumer + | undefined; + +export function registerLangSmithInstrumentation( + options: LangSmithInstrumentationOptions = {}, +): void { + langSmithInstrumentationConsumer ??= new LangSmithInstrumentationConsumer( + options, + ); + langSmithInstrumentationConsumer.register(); +} + function isRecord(value: unknown): value is Record { return typeof value === "object" && value !== null && !Array.isArray(value); } diff --git a/js/src/instrumentation/plugins/mistral-channels.ts b/js/src/instrumentation/providers/mistral-channels.ts similarity index 100% rename from js/src/instrumentation/plugins/mistral-channels.ts rename to js/src/instrumentation/providers/mistral-channels.ts diff --git a/js/src/instrumentation/plugins/mistral-plugin.test.ts b/js/src/instrumentation/providers/mistral-instrumentation.test.ts similarity index 99% rename from js/src/instrumentation/plugins/mistral-plugin.test.ts rename to js/src/instrumentation/providers/mistral-instrumentation.test.ts index 96d410440..adab7b87b 100644 --- a/js/src/instrumentation/plugins/mistral-plugin.test.ts +++ b/js/src/instrumentation/providers/mistral-instrumentation.test.ts @@ -4,7 +4,7 @@ import { extractMistralRequestMetadata, extractMistralResponseMetadata, parseMistralMetricsFromUsage, -} from "./mistral-plugin"; +} from "./mistral-instrumentation"; describe("extractMistralRequestMetadata", () => { it("keeps only allowlisted request metadata", () => { diff --git a/js/src/instrumentation/plugins/mistral-plugin.ts b/js/src/instrumentation/providers/mistral-instrumentation.ts similarity index 77% rename from js/src/instrumentation/plugins/mistral-plugin.ts rename to js/src/instrumentation/providers/mistral-instrumentation.ts index f33f82554..7179b569d 100644 --- a/js/src/instrumentation/plugins/mistral-plugin.ts +++ b/js/src/instrumentation/providers/mistral-instrumentation.ts @@ -1,8 +1,6 @@ -import { BasePlugin } from "../core"; import { traceAsyncChannel, traceStreamingChannel, - unsubscribeAll, } from "../core/channel-tracing"; import { SpanTypeAttribute, isObject } from "../../../util/index"; import { processInputAttachments } from "../../wrappers/attachment-utils"; @@ -19,156 +17,130 @@ import type { MistralToolCallDelta, } from "../../vendor-sdk-types/mistral"; -export class MistralPlugin extends BasePlugin { - protected onEnable(): void { +class MistralInstrumentationConsumer { + public register(): void { this.subscribeToMistralChannels(); } - protected onDisable(): void { - this.unsubscribers = unsubscribeAll(this.unsubscribers); - } - private subscribeToMistralChannels(): void { - this.unsubscribers.push( - traceStreamingChannel(mistralChannels.chatComplete, { - name: "mistral.chat.complete", - type: SpanTypeAttribute.LLM, - extractInput: extractMessagesInputWithMetadata, - extractOutput: (result) => { - return result?.choices; - }, - extractMetadata: (result) => extractMistralResponseMetadata(result), - extractMetrics: (result, startTime) => - extractMistralMetrics(result?.usage, startTime), - }), - ); - - this.unsubscribers.push( - traceStreamingChannel(mistralChannels.chatStream, { - name: "mistral.chat.stream", - type: SpanTypeAttribute.LLM, - extractInput: extractMessagesInputWithMetadata, - extractOutput: extractMistralStreamOutput, - extractMetadata: (result) => extractMistralResponseMetadata(result), - extractMetrics: (result, startTime) => - extractMistralStreamingMetrics(result, startTime), - aggregateChunks: aggregateMistralStreamChunks, - }), - ); - - this.unsubscribers.push( - traceAsyncChannel(mistralChannels.embeddingsCreate, { - name: "mistral.embeddings.create", - type: SpanTypeAttribute.LLM, - extractInput: extractEmbeddingInputWithMetadata, - extractOutput: (result) => { - const embedding = result?.data?.[0]?.embedding; - return Array.isArray(embedding) - ? { embedding_length: embedding.length } - : undefined; - }, - extractMetadata: (result) => extractMistralResponseMetadata(result), - extractMetrics: (result) => parseMistralMetricsFromUsage(result?.usage), - }), - ); - - this.unsubscribers.push( - traceAsyncChannel(mistralChannels.classifiersModerate, { - name: "mistral.classifiers.moderate", - type: SpanTypeAttribute.LLM, - extractInput: extractClassifierInputWithMetadata, - extractOutput: extractClassifierOutput, - extractMetadata: (result) => extractMistralResponseMetadata(result), - extractMetrics: (result) => parseMistralMetricsFromUsage(result?.usage), - }), - ); - - this.unsubscribers.push( - traceAsyncChannel(mistralChannels.classifiersModerateChat, { - name: "mistral.classifiers.moderateChat", - type: SpanTypeAttribute.LLM, - extractInput: extractClassifierInputWithMetadata, - extractOutput: extractClassifierOutput, - extractMetadata: (result) => extractMistralResponseMetadata(result), - extractMetrics: (result) => parseMistralMetricsFromUsage(result?.usage), - }), - ); - - this.unsubscribers.push( - traceAsyncChannel(mistralChannels.classifiersClassify, { - name: "mistral.classifiers.classify", - type: SpanTypeAttribute.LLM, - extractInput: extractClassifierInputWithMetadata, - extractOutput: extractClassifierOutput, - extractMetadata: (result) => extractMistralResponseMetadata(result), - extractMetrics: (result) => parseMistralMetricsFromUsage(result?.usage), - }), - ); - - this.unsubscribers.push( - traceAsyncChannel(mistralChannels.classifiersClassifyChat, { - name: "mistral.classifiers.classifyChat", - type: SpanTypeAttribute.LLM, - extractInput: extractClassifierInputWithMetadata, - extractOutput: extractClassifierOutput, - extractMetadata: (result) => extractMistralResponseMetadata(result), - extractMetrics: (result) => parseMistralMetricsFromUsage(result?.usage), - }), - ); - - this.unsubscribers.push( - traceStreamingChannel(mistralChannels.fimComplete, { - name: "mistral.fim.complete", - type: SpanTypeAttribute.LLM, - extractInput: extractPromptInputWithMetadata, - extractOutput: (result) => { - return result?.choices; - }, - extractMetadata: (result) => extractMistralResponseMetadata(result), - extractMetrics: (result, startTime) => - extractMistralMetrics(result?.usage, startTime), - }), - ); - - this.unsubscribers.push( - traceStreamingChannel(mistralChannels.fimStream, { - name: "mistral.fim.stream", - type: SpanTypeAttribute.LLM, - extractInput: extractPromptInputWithMetadata, - extractOutput: extractMistralStreamOutput, - extractMetadata: (result) => extractMistralResponseMetadata(result), - extractMetrics: (result, startTime) => - extractMistralStreamingMetrics(result, startTime), - aggregateChunks: aggregateMistralStreamChunks, - }), - ); - - this.unsubscribers.push( - traceStreamingChannel(mistralChannels.agentsComplete, { - name: "mistral.agents.complete", - type: SpanTypeAttribute.LLM, - extractInput: extractMessagesInputWithMetadata, - extractOutput: (result) => { - return result?.choices; - }, - extractMetadata: (result) => extractMistralResponseMetadata(result), - extractMetrics: (result, startTime) => - extractMistralMetrics(result?.usage, startTime), - }), - ); - - this.unsubscribers.push( - traceStreamingChannel(mistralChannels.agentsStream, { - name: "mistral.agents.stream", - type: SpanTypeAttribute.LLM, - extractInput: extractMessagesInputWithMetadata, - extractOutput: extractMistralStreamOutput, - extractMetadata: (result) => extractMistralResponseMetadata(result), - extractMetrics: (result, startTime) => - extractMistralStreamingMetrics(result, startTime), - aggregateChunks: aggregateMistralStreamChunks, - }), - ); + traceStreamingChannel(mistralChannels.chatComplete, { + name: "mistral.chat.complete", + type: SpanTypeAttribute.LLM, + extractInput: extractMessagesInputWithMetadata, + extractOutput: (result) => { + return result?.choices; + }, + extractMetadata: (result) => extractMistralResponseMetadata(result), + extractMetrics: (result, startTime) => + extractMistralMetrics(result?.usage, startTime), + }); + + traceStreamingChannel(mistralChannels.chatStream, { + name: "mistral.chat.stream", + type: SpanTypeAttribute.LLM, + extractInput: extractMessagesInputWithMetadata, + extractOutput: extractMistralStreamOutput, + extractMetadata: (result) => extractMistralResponseMetadata(result), + extractMetrics: (result, startTime) => + extractMistralStreamingMetrics(result, startTime), + aggregateChunks: aggregateMistralStreamChunks, + }); + + traceAsyncChannel(mistralChannels.embeddingsCreate, { + name: "mistral.embeddings.create", + type: SpanTypeAttribute.LLM, + extractInput: extractEmbeddingInputWithMetadata, + extractOutput: (result) => { + const embedding = result?.data?.[0]?.embedding; + return Array.isArray(embedding) + ? { embedding_length: embedding.length } + : undefined; + }, + extractMetadata: (result) => extractMistralResponseMetadata(result), + extractMetrics: (result) => parseMistralMetricsFromUsage(result?.usage), + }); + + traceAsyncChannel(mistralChannels.classifiersModerate, { + name: "mistral.classifiers.moderate", + type: SpanTypeAttribute.LLM, + extractInput: extractClassifierInputWithMetadata, + extractOutput: extractClassifierOutput, + extractMetadata: (result) => extractMistralResponseMetadata(result), + extractMetrics: (result) => parseMistralMetricsFromUsage(result?.usage), + }); + + traceAsyncChannel(mistralChannels.classifiersModerateChat, { + name: "mistral.classifiers.moderateChat", + type: SpanTypeAttribute.LLM, + extractInput: extractClassifierInputWithMetadata, + extractOutput: extractClassifierOutput, + extractMetadata: (result) => extractMistralResponseMetadata(result), + extractMetrics: (result) => parseMistralMetricsFromUsage(result?.usage), + }); + + traceAsyncChannel(mistralChannels.classifiersClassify, { + name: "mistral.classifiers.classify", + type: SpanTypeAttribute.LLM, + extractInput: extractClassifierInputWithMetadata, + extractOutput: extractClassifierOutput, + extractMetadata: (result) => extractMistralResponseMetadata(result), + extractMetrics: (result) => parseMistralMetricsFromUsage(result?.usage), + }); + + traceAsyncChannel(mistralChannels.classifiersClassifyChat, { + name: "mistral.classifiers.classifyChat", + type: SpanTypeAttribute.LLM, + extractInput: extractClassifierInputWithMetadata, + extractOutput: extractClassifierOutput, + extractMetadata: (result) => extractMistralResponseMetadata(result), + extractMetrics: (result) => parseMistralMetricsFromUsage(result?.usage), + }); + + traceStreamingChannel(mistralChannels.fimComplete, { + name: "mistral.fim.complete", + type: SpanTypeAttribute.LLM, + extractInput: extractPromptInputWithMetadata, + extractOutput: (result) => { + return result?.choices; + }, + extractMetadata: (result) => extractMistralResponseMetadata(result), + extractMetrics: (result, startTime) => + extractMistralMetrics(result?.usage, startTime), + }); + + traceStreamingChannel(mistralChannels.fimStream, { + name: "mistral.fim.stream", + type: SpanTypeAttribute.LLM, + extractInput: extractPromptInputWithMetadata, + extractOutput: extractMistralStreamOutput, + extractMetadata: (result) => extractMistralResponseMetadata(result), + extractMetrics: (result, startTime) => + extractMistralStreamingMetrics(result, startTime), + aggregateChunks: aggregateMistralStreamChunks, + }); + + traceStreamingChannel(mistralChannels.agentsComplete, { + name: "mistral.agents.complete", + type: SpanTypeAttribute.LLM, + extractInput: extractMessagesInputWithMetadata, + extractOutput: (result) => { + return result?.choices; + }, + extractMetadata: (result) => extractMistralResponseMetadata(result), + extractMetrics: (result, startTime) => + extractMistralMetrics(result?.usage, startTime), + }); + + traceStreamingChannel(mistralChannels.agentsStream, { + name: "mistral.agents.stream", + type: SpanTypeAttribute.LLM, + extractInput: extractMessagesInputWithMetadata, + extractOutput: extractMistralStreamOutput, + extractMetadata: (result) => extractMistralResponseMetadata(result), + extractMetrics: (result, startTime) => + extractMistralStreamingMetrics(result, startTime), + aggregateChunks: aggregateMistralStreamChunks, + }); } } @@ -909,3 +881,10 @@ export function aggregateMistralStreamChunks( ...(metadata ? { metadata } : {}), }; } + +let mistralInstrumentationConsumer: MistralInstrumentationConsumer | undefined; + +export function registerMistralInstrumentation(): void { + mistralInstrumentationConsumer ??= new MistralInstrumentationConsumer(); + mistralInstrumentationConsumer.register(); +} diff --git a/js/src/instrumentation/plugins/ollama-channels.ts b/js/src/instrumentation/providers/ollama-channels.ts similarity index 100% rename from js/src/instrumentation/plugins/ollama-channels.ts rename to js/src/instrumentation/providers/ollama-channels.ts diff --git a/js/src/instrumentation/plugins/ollama-plugin.test.ts b/js/src/instrumentation/providers/ollama-instrumentation.test.ts similarity index 99% rename from js/src/instrumentation/plugins/ollama-plugin.test.ts rename to js/src/instrumentation/providers/ollama-instrumentation.test.ts index 0006efea8..4503d4ddf 100644 --- a/js/src/instrumentation/plugins/ollama-plugin.test.ts +++ b/js/src/instrumentation/providers/ollama-instrumentation.test.ts @@ -9,7 +9,7 @@ import { extractOllamaEmbedOutput, extractOllamaGenerateInput, extractOllamaMetrics, -} from "./ollama-plugin"; +} from "./ollama-instrumentation"; configureNode(); diff --git a/js/src/instrumentation/plugins/ollama-plugin.ts b/js/src/instrumentation/providers/ollama-instrumentation.ts similarity index 91% rename from js/src/instrumentation/plugins/ollama-plugin.ts rename to js/src/instrumentation/providers/ollama-instrumentation.ts index f48040348..405b3dff9 100644 --- a/js/src/instrumentation/plugins/ollama-plugin.ts +++ b/js/src/instrumentation/providers/ollama-instrumentation.ts @@ -14,54 +14,44 @@ import type { OllamaToolCall, OllamaUsageResponse, } from "../../vendor-sdk-types/ollama"; -import { BasePlugin } from "../core"; import type { AsyncEndOf } from "../core/channel-definitions"; import { traceAsyncChannel, traceStreamingChannel, - unsubscribeAll, } from "../core/channel-tracing"; import { ollamaChannels } from "./ollama-channels"; -export class OllamaPlugin extends BasePlugin { - protected onEnable(): void { - this.unsubscribers.push( - traceStreamingChannel(ollamaChannels.chat, { - name: "ollama.chat", - type: SpanTypeAttribute.LLM, - extractInput: extractOllamaChatInput, - extractOutput: (result, event) => - extractOllamaChatOutput( - result, - countOllamaToolCalls(event?.arguments?.[0]?.messages), - ), - extractMetadata: extractOllamaResponseMetadata, - extractMetrics: extractOllamaMetrics, - aggregateChunks: aggregateOllamaChatChunks, - }), - traceStreamingChannel(ollamaChannels.generate, { - name: "ollama.generate", - type: SpanTypeAttribute.LLM, - extractInput: extractOllamaGenerateInput, - extractOutput: extractOllamaGenerateOutput, - extractMetadata: extractOllamaResponseMetadata, - extractMetrics: extractOllamaMetrics, - aggregateChunks: aggregateOllamaGenerateChunks, - }), - traceAsyncChannel(ollamaChannels.embed, { - name: "ollama.embed", - type: SpanTypeAttribute.LLM, - extractInput: extractOllamaEmbedInput, - extractOutput: extractOllamaEmbedOutput, - extractMetadata: extractOllamaResponseMetadata, - extractMetrics: extractOllamaMetrics, - }), - ); - } - - protected onDisable(): void { - this.unsubscribers = unsubscribeAll(this.unsubscribers); - } +export function registerOllamaInstrumentation(): void { + traceStreamingChannel(ollamaChannels.chat, { + name: "ollama.chat", + type: SpanTypeAttribute.LLM, + extractInput: extractOllamaChatInput, + extractOutput: (result, event) => + extractOllamaChatOutput( + result, + countOllamaToolCalls(event?.arguments?.[0]?.messages), + ), + extractMetadata: extractOllamaResponseMetadata, + extractMetrics: extractOllamaMetrics, + aggregateChunks: aggregateOllamaChatChunks, + }); + traceStreamingChannel(ollamaChannels.generate, { + name: "ollama.generate", + type: SpanTypeAttribute.LLM, + extractInput: extractOllamaGenerateInput, + extractOutput: extractOllamaGenerateOutput, + extractMetadata: extractOllamaResponseMetadata, + extractMetrics: extractOllamaMetrics, + aggregateChunks: aggregateOllamaGenerateChunks, + }); + traceAsyncChannel(ollamaChannels.embed, { + name: "ollama.embed", + type: SpanTypeAttribute.LLM, + extractInput: extractOllamaEmbedInput, + extractOutput: extractOllamaEmbedOutput, + extractMetadata: extractOllamaResponseMetadata, + extractMetrics: extractOllamaMetrics, + }); } function isNonNegativeNumber(value: unknown): value is number { diff --git a/js/src/instrumentation/plugins/openai-agents-channels.ts b/js/src/instrumentation/providers/openai-agents-channels.ts similarity index 100% rename from js/src/instrumentation/plugins/openai-agents-channels.ts rename to js/src/instrumentation/providers/openai-agents-channels.ts diff --git a/js/src/instrumentation/plugins/openai-agents-plugin.test.ts b/js/src/instrumentation/providers/openai-agents-instrumentation.test.ts similarity index 88% rename from js/src/instrumentation/plugins/openai-agents-plugin.test.ts rename to js/src/instrumentation/providers/openai-agents-instrumentation.test.ts index 9f2a604bd..fb6f62fba 100644 --- a/js/src/instrumentation/plugins/openai-agents-plugin.test.ts +++ b/js/src/instrumentation/providers/openai-agents-instrumentation.test.ts @@ -1,37 +1,36 @@ import { afterEach, beforeAll, beforeEach, describe, expect, it } from "vitest"; import { configureNode } from "../../node/config"; import { _exportsForTestingOnly, initLogger } from "../../logger"; +import { configureInstrumentation } from "../registry"; import { openAIAgentsCoreChannels } from "./openai-agents-channels"; -import { OpenAIAgentsPlugin } from "./openai-agents-plugin"; +import { registerOpenAIAgentsInstrumentation } from "./openai-agents-instrumentation"; try { + configureInstrumentation({ integrations: { openAIAgents: false } }); configureNode(); } catch { // Best-effort initialization for test environments. } -describe("OpenAIAgentsPlugin", () => { +describe("registerOpenAIAgentsInstrumentation", () => { let backgroundLogger: ReturnType< typeof _exportsForTestingOnly.useTestBackgroundLogger >; - let plugin: OpenAIAgentsPlugin; beforeAll(async () => { await _exportsForTestingOnly.simulateLoginForTests(); + registerOpenAIAgentsInstrumentation(); }); beforeEach(() => { backgroundLogger = _exportsForTestingOnly.useTestBackgroundLogger(); initLogger({ - projectName: "openai-agents-plugin.test.ts", + projectName: "openai-agents-instrumentation.test.ts", projectId: "test-project-id", }); - plugin = new OpenAIAgentsPlugin(); - plugin.enable(); }); afterEach(() => { - plugin.disable(); _exportsForTestingOnly.clearTestBackgroundLogger(); }); diff --git a/js/src/instrumentation/plugins/openai-agents-plugin.ts b/js/src/instrumentation/providers/openai-agents-instrumentation.ts similarity index 80% rename from js/src/instrumentation/plugins/openai-agents-plugin.ts rename to js/src/instrumentation/providers/openai-agents-instrumentation.ts index 4cf8af1a0..9db90ac4c 100644 --- a/js/src/instrumentation/plugins/openai-agents-plugin.ts +++ b/js/src/instrumentation/providers/openai-agents-instrumentation.ts @@ -1,5 +1,3 @@ -import { BasePlugin } from "../core"; -import { unsubscribeAll } from "../core/channel-tracing"; import { isObject } from "../../../util/index"; import { openAIAgentsCoreChannels } from "./openai-agents-channels"; import { OpenAIAgentsTraceProcessor } from "./openai-agents-trace-processor"; @@ -41,18 +39,13 @@ function isOpenAIAgentsSpan(value: unknown): value is OpenAIAgentsSpan { ); } -export class OpenAIAgentsPlugin extends BasePlugin { +class OpenAIAgentsInstrumentationConsumer { private processor = new OpenAIAgentsTraceProcessor(); - protected onEnable(): void { + public register(): void { this.subscribeToTraceLifecycle(); } - protected onDisable(): void { - this.unsubscribers = unsubscribeAll(this.unsubscribers); - void this.processor.shutdown(); - } - private subscribeToTraceLifecycle(): void { const traceStartChannel = openAIAgentsCoreChannels.onTraceStart.tracingChannel(); @@ -65,9 +58,6 @@ export class OpenAIAgentsPlugin extends BasePlugin { }, }; traceStartChannel.subscribe(traceStartHandlers); - this.unsubscribers.push(() => - traceStartChannel.unsubscribe(traceStartHandlers), - ); const traceEndChannel = openAIAgentsCoreChannels.onTraceEnd.tracingChannel(); @@ -80,9 +70,6 @@ export class OpenAIAgentsPlugin extends BasePlugin { }, }; traceEndChannel.subscribe(traceEndHandlers); - this.unsubscribers.push(() => - traceEndChannel.unsubscribe(traceEndHandlers), - ); const spanStartChannel = openAIAgentsCoreChannels.onSpanStart.tracingChannel(); @@ -95,9 +82,6 @@ export class OpenAIAgentsPlugin extends BasePlugin { }, }; spanStartChannel.subscribe(spanStartHandlers); - this.unsubscribers.push(() => - spanStartChannel.unsubscribe(spanStartHandlers), - ); const spanEndChannel = openAIAgentsCoreChannels.onSpanEnd.tracingChannel(); const spanEndHandlers = { @@ -109,6 +93,15 @@ export class OpenAIAgentsPlugin extends BasePlugin { }, }; spanEndChannel.subscribe(spanEndHandlers); - this.unsubscribers.push(() => spanEndChannel.unsubscribe(spanEndHandlers)); } } + +let openAIAgentsInstrumentationConsumer: + | OpenAIAgentsInstrumentationConsumer + | undefined; + +export function registerOpenAIAgentsInstrumentation(): void { + openAIAgentsInstrumentationConsumer ??= + new OpenAIAgentsInstrumentationConsumer(); + openAIAgentsInstrumentationConsumer.register(); +} diff --git a/js/src/instrumentation/plugins/openai-agents-trace-processor.ts b/js/src/instrumentation/providers/openai-agents-trace-processor.ts similarity index 100% rename from js/src/instrumentation/plugins/openai-agents-trace-processor.ts rename to js/src/instrumentation/providers/openai-agents-trace-processor.ts diff --git a/js/src/instrumentation/plugins/openai-channels.ts b/js/src/instrumentation/providers/openai-channels.ts similarity index 100% rename from js/src/instrumentation/plugins/openai-channels.ts rename to js/src/instrumentation/providers/openai-channels.ts diff --git a/js/src/instrumentation/plugins/openai-codex-channels.ts b/js/src/instrumentation/providers/openai-codex-channels.ts similarity index 100% rename from js/src/instrumentation/plugins/openai-codex-channels.ts rename to js/src/instrumentation/providers/openai-codex-channels.ts diff --git a/js/src/instrumentation/plugins/openai-codex-plugin.test.ts b/js/src/instrumentation/providers/openai-codex-instrumentation.test.ts similarity index 92% rename from js/src/instrumentation/plugins/openai-codex-plugin.test.ts rename to js/src/instrumentation/providers/openai-codex-instrumentation.test.ts index 932abe0ba..f7250bb3b 100644 --- a/js/src/instrumentation/plugins/openai-codex-plugin.test.ts +++ b/js/src/instrumentation/providers/openai-codex-instrumentation.test.ts @@ -15,11 +15,11 @@ vi.mock("../../logger", () => ({ })); import iso from "../../isomorph"; -import { OpenAICodexPlugin } from "./openai-codex-plugin"; +import { registerOpenAICodexInstrumentation } from "./openai-codex-instrumentation"; const mockNewTracingChannel = iso.newTracingChannel as ReturnType; -describe("OpenAICodexPlugin", () => { +describe("registerOpenAICodexInstrumentation", () => { let handlersByName: Map; let spans: Array<{ end: ReturnType; @@ -33,7 +33,6 @@ describe("OpenAICodexPlugin", () => { spans = []; mockNewTracingChannel.mockImplementation((name: string) => ({ subscribe: vi.fn((handlers) => handlersByName.set(name, handlers)), - unsubscribe: vi.fn(), })); mockStartSpan.mockImplementation((args: any) => { const span = { @@ -55,8 +54,7 @@ describe("OpenAICodexPlugin", () => { }); it("uses explicit token totals by precedence instead of summing breakdowns", async () => { - const plugin = new OpenAICodexPlugin(); - plugin.enable(); + registerOpenAICodexInstrumentation(); const runHandlers = handlersByName.get( "orchestrion:@openai/codex-sdk:Thread.run", diff --git a/js/src/instrumentation/plugins/openai-codex-plugin.ts b/js/src/instrumentation/providers/openai-codex-instrumentation.ts similarity index 98% rename from js/src/instrumentation/plugins/openai-codex-plugin.ts rename to js/src/instrumentation/providers/openai-codex-instrumentation.ts index 4005ecb86..abf442120 100644 --- a/js/src/instrumentation/plugins/openai-codex-plugin.ts +++ b/js/src/instrumentation/providers/openai-codex-instrumentation.ts @@ -1,4 +1,4 @@ -import { BasePlugin, toLoggedError } from "../core"; +import { toLoggedError } from "../core"; import type { ChannelMessage } from "../core/channel-definitions"; import type { IsoChannelHandlers } from "../../isomorph"; import { debugLogger } from "../../debug-logger"; @@ -55,19 +55,12 @@ const PATCHED_STREAMED_TURN = Symbol.for( "braintrust.openai-codex.patched-streamed-turn", ); -export class OpenAICodexPlugin extends BasePlugin { - protected onEnable(): void { +class OpenAICodexInstrumentationConsumer { + public register(): void { this.subscribeToRun(); this.subscribeToRunStreamed(); } - protected onDisable(): void { - for (const unsubscribe of this.unsubscribers) { - unsubscribe(); - } - this.unsubscribers = []; - } - private subscribeToRun(): void { const channel = openAICodexChannels.run.tracingChannel(); const states = new WeakMap(); @@ -97,9 +90,6 @@ export class OpenAICodexPlugin extends BasePlugin { }; channel.subscribe(handlers); - this.unsubscribers.push(() => { - channel.unsubscribe(handlers); - }); } private subscribeToRunStreamed(): void { @@ -131,9 +121,6 @@ export class OpenAICodexPlugin extends BasePlugin { }; channel.subscribe(handlers); - this.unsubscribers.push(() => { - channel.unsubscribe(handlers); - }); } } @@ -941,3 +928,13 @@ function safeLog(span: Span, event: Parameters[0]): void { function logInstrumentationError(context: string, error: unknown): void { debugLogger.error(`Error processing ${context}:`, error); } + +let openAICodexInstrumentationConsumer: + | OpenAICodexInstrumentationConsumer + | undefined; + +export function registerOpenAICodexInstrumentation(): void { + openAICodexInstrumentationConsumer ??= + new OpenAICodexInstrumentationConsumer(); + openAICodexInstrumentationConsumer.register(); +} diff --git a/js/src/instrumentation/plugins/openai-plugin.test.ts b/js/src/instrumentation/providers/openai-instrumentation.test.ts similarity index 99% rename from js/src/instrumentation/plugins/openai-plugin.test.ts rename to js/src/instrumentation/providers/openai-instrumentation.test.ts index 5f2b237fa..d51301e85 100644 --- a/js/src/instrumentation/plugins/openai-plugin.test.ts +++ b/js/src/instrumentation/providers/openai-instrumentation.test.ts @@ -3,7 +3,7 @@ import { parseMetricsFromUsage, processImagesInOutput, aggregateChatCompletionChunks, -} from "./openai-plugin"; +} from "./openai-instrumentation"; import { Attachment } from "../../logger"; describe("parseMetricsFromUsage", () => { diff --git a/js/src/instrumentation/providers/openai-instrumentation.ts b/js/src/instrumentation/providers/openai-instrumentation.ts new file mode 100644 index 000000000..30cddb5c2 --- /dev/null +++ b/js/src/instrumentation/providers/openai-instrumentation.ts @@ -0,0 +1,573 @@ +import { + traceAsyncChannel, + traceStreamingChannel, + traceSyncStreamChannel, +} from "../core/channel-tracing"; +import { Attachment } from "../../logger"; +import { SpanTypeAttribute, isObject } from "../../../util/index"; +import { getCurrentUnixTimestamp } from "../../util"; +import { processInputAttachments } from "../../wrappers/attachment-utils"; +import { openAIChannels } from "./openai-channels"; +import { + BRAINTRUST_CACHED_STREAM_METRIC, + getCachedMetricFromHeaders, + parseMetricsFromUsage, +} from "../../openai-utils"; +import type { + OpenAIChatChoice, + OpenAIChatCompletionChunk, + OpenAIChatLogprobs, + OpenAIResponseStreamEvent, +} from "../../vendor-sdk-types/openai"; + +export function registerOpenAIInstrumentation(): void { + // Chat Completions - supports streaming + traceStreamingChannel(openAIChannels.chatCompletionsCreate, { + name: "Chat Completion", + type: SpanTypeAttribute.LLM, + extractInput: ([params]) => { + const { messages, ...metadata } = params; + return { + input: processInputAttachments(messages), + metadata: { ...metadata, provider: "openai" }, + }; + }, + extractOutput: (result) => { + return result?.choices; + }, + extractMetrics: (result, startTime, endEvent) => { + const metrics = withCachedMetric( + parseMetricsFromUsage(result?.usage), + result, + endEvent, + ); + if (startTime) { + metrics.time_to_first_token = getCurrentUnixTimestamp() - startTime; + } + return metrics; + }, + aggregateChunks: aggregateChatCompletionChunks, + }); + + // Embeddings + traceAsyncChannel(openAIChannels.embeddingsCreate, { + name: "Embedding", + type: SpanTypeAttribute.LLM, + extractInput: ([params]) => { + const { input, ...metadata } = params; + return { + input, + metadata: { ...metadata, provider: "openai" }, + }; + }, + extractOutput: (result) => { + const embedding = result?.data?.[0]?.embedding; + return Array.isArray(embedding) + ? { embedding_length: embedding.length } + : undefined; + }, + extractMetrics: (result, _startTime, endEvent) => { + return withCachedMetric( + parseMetricsFromUsage(result?.usage), + result, + endEvent, + ); + }, + }); + + // Beta Chat Completions Parse + traceStreamingChannel(openAIChannels.betaChatCompletionsParse, { + name: "Chat Completion", + type: SpanTypeAttribute.LLM, + extractInput: ([params]) => { + const { messages, ...metadata } = params; + return { + input: processInputAttachments(messages), + metadata: { ...metadata, provider: "openai" }, + }; + }, + extractOutput: (result) => { + return result?.choices; + }, + extractMetrics: (result, startTime, endEvent) => { + const metrics = withCachedMetric( + parseMetricsFromUsage(result?.usage), + result, + endEvent, + ); + if (startTime) { + metrics.time_to_first_token = getCurrentUnixTimestamp() - startTime; + } + return metrics; + }, + aggregateChunks: aggregateChatCompletionChunks, + }); + + // Beta Chat Completions Stream (sync method returning event-based stream) + traceSyncStreamChannel(openAIChannels.betaChatCompletionsStream, { + name: "Chat Completion", + type: SpanTypeAttribute.LLM, + extractInput: ([params]) => { + const { messages, ...metadata } = params; + return { + input: processInputAttachments(messages), + metadata: { ...metadata, provider: "openai" }, + }; + }, + }); + + // Moderations + traceAsyncChannel(openAIChannels.moderationsCreate, { + name: "Moderation", + type: SpanTypeAttribute.LLM, + extractInput: ([params]) => { + const { input, ...metadata } = params; + return { + input, + metadata: { ...metadata, provider: "openai" }, + }; + }, + extractOutput: (result) => { + return result?.results; + }, + extractMetrics: (result, _startTime, endEvent) => { + return withCachedMetric( + parseMetricsFromUsage(result?.usage), + result, + endEvent, + ); + }, + }); + + // Responses API - create (supports streaming via stream=true param) + traceStreamingChannel(openAIChannels.responsesCreate, { + name: "openai.responses.create", + type: SpanTypeAttribute.LLM, + extractInput: ([params]) => { + const { input, ...metadata } = params; + return { + input: processInputAttachments(input), + metadata: { ...metadata, provider: "openai" }, + }; + }, + extractOutput: (result) => { + return processImagesInOutput(result?.output); + }, + extractMetadata: (result) => { + if (!result) { + return undefined; + } + const { output: _output, usage: _usage, ...metadata } = result; + return Object.keys(metadata).length > 0 ? metadata : undefined; + }, + extractMetrics: (result, startTime, endEvent) => { + const metrics = withCachedMetric( + parseMetricsFromUsage(result?.usage), + result, + endEvent, + ); + if (startTime) { + metrics.time_to_first_token = getCurrentUnixTimestamp() - startTime; + } + return metrics; + }, + aggregateChunks: aggregateResponseStreamEvents, + }); + + // Responses API - stream (sync method returning event-based stream) + traceSyncStreamChannel(openAIChannels.responsesStream, { + name: "openai.responses.create", + type: SpanTypeAttribute.LLM, + extractInput: ([params]) => { + const { input, ...metadata } = params; + return { + input: processInputAttachments(input), + metadata: { ...metadata, provider: "openai" }, + }; + }, + extractFromEvent: (event) => { + if (event.type !== "response.completed" || !event.response) { + return {}; + } + + const response = event.response; + const data: Record = {}; + + if (response.output !== undefined) { + data.output = processImagesInOutput(response.output); + } + + const { usage: _usage, output: _output, ...metadata } = response; + if (Object.keys(metadata).length > 0) { + data.metadata = metadata; + } + + data.metrics = parseMetricsFromUsage(response.usage); + return data; + }, + }); + + // Responses API - parse + traceStreamingChannel(openAIChannels.responsesParse, { + name: "openai.responses.parse", + type: SpanTypeAttribute.LLM, + extractInput: ([params]) => { + const { input, ...metadata } = params; + return { + input: processInputAttachments(input), + metadata: { ...metadata, provider: "openai" }, + }; + }, + extractOutput: (result) => { + return processImagesInOutput(result?.output); + }, + extractMetadata: (result) => { + if (!result) { + return undefined; + } + const { output: _output, usage: _usage, ...metadata } = result; + return Object.keys(metadata).length > 0 ? metadata : undefined; + }, + extractMetrics: (result, startTime, endEvent) => { + const metrics = withCachedMetric( + parseMetricsFromUsage(result?.usage), + result, + endEvent, + ); + if (startTime) { + metrics.time_to_first_token = getCurrentUnixTimestamp() - startTime; + } + return metrics; + }, + aggregateChunks: aggregateResponseStreamEvents, + }); + + // Responses API - compact + traceAsyncChannel(openAIChannels.responsesCompact, { + name: "openai.responses.compact", + type: SpanTypeAttribute.LLM, + extractInput: ([params]) => { + const { input, ...metadata } = params; + return { + input: processInputAttachments(input), + metadata: { ...metadata, provider: "openai" }, + }; + }, + extractOutput: (result) => { + return processImagesInOutput(result?.output); + }, + extractMetadata: (result) => { + if (!result) { + return undefined; + } + const { output: _output, usage: _usage, ...metadata } = result; + return Object.keys(metadata).length > 0 ? metadata : undefined; + }, + extractMetrics: (result, startTime, endEvent) => { + const metrics = withCachedMetric( + parseMetricsFromUsage(result?.usage), + result, + endEvent, + ); + if (startTime) { + metrics.time_to_first_token = getCurrentUnixTimestamp() - startTime; + } + return metrics; + }, + }); +} + +function getCachedMetricFromEndEvent(endEvent: unknown): number | undefined { + if (!isObject(endEvent)) { + return undefined; + } + + const response = (endEvent as Record).response; + if (!isObject(response)) { + return undefined; + } + + const headers = (response as { headers?: unknown }).headers; + if (!headers || typeof (headers as Headers).get !== "function") { + return undefined; + } + + return getCachedMetricFromHeaders(headers as Headers); +} + +function withCachedMetric( + metrics: Record, + result: unknown, + endEvent?: unknown, +): Record { + if (metrics.cached !== undefined) { + return metrics; + } + + const cachedFromEvent = getCachedMetricFromEndEvent(endEvent); + if (cachedFromEvent !== undefined) { + return { + ...metrics, + cached: cachedFromEvent, + }; + } + + if (!isObject(result)) { + return metrics; + } + + const cached = (result as Record)[ + BRAINTRUST_CACHED_STREAM_METRIC + ]; + + if (typeof cached !== "number") { + return metrics; + } + + return { + ...metrics, + cached, + }; +} + +/** + * Process output to convert base64 images to attachments. + * Used for Responses API image generation output. + */ +export function processImagesInOutput(output: any): any { + if (Array.isArray(output)) { + return output.map(processImagesInOutput); + } + + if (isObject(output)) { + if ( + output.type === "image_generation_call" && + output.result && + typeof output.result === "string" + ) { + const fileExtension = output.output_format || "png"; + const contentType = `image/${fileExtension}`; + + const baseFilename = + output.revised_prompt && typeof output.revised_prompt === "string" + ? output.revised_prompt.slice(0, 50).replace(/[^a-zA-Z0-9]/g, "_") + : "generated_image"; + const filename = `${baseFilename}.${fileExtension}`; + + // Convert base64 string to Blob + const binaryString = atob(output.result); + const bytes = new Uint8Array(binaryString.length); + for (let i = 0; i < binaryString.length; i++) { + bytes[i] = binaryString.charCodeAt(i); + } + const blob = new Blob([bytes], { type: contentType }); + + const attachment = new Attachment({ + data: blob, + filename: filename, + contentType: contentType, + }); + + return { + ...output, + result: attachment, + }; + } + } + + return output; +} + +function mergeLogprobTokens( + existing: OpenAIChatLogprobs["content"] | OpenAIChatLogprobs["refusal"], + incoming: OpenAIChatLogprobs["content"] | OpenAIChatLogprobs["refusal"], +): OpenAIChatLogprobs["content"] | OpenAIChatLogprobs["refusal"] { + if (incoming === undefined) { + return existing; + } + + if (incoming === null) { + return existing ?? null; + } + + if (Array.isArray(existing)) { + return [...existing, ...incoming]; + } + + return [...incoming]; +} + +function aggregateChatLogprobs( + existing: OpenAIChatLogprobs | null | undefined, + incoming: OpenAIChatLogprobs | null | undefined, +): OpenAIChatLogprobs | null | undefined { + if (incoming === undefined) { + return existing; + } + + if (incoming === null) { + return existing ?? null; + } + + const aggregated: OpenAIChatLogprobs = + existing && existing !== null + ? { ...existing, ...incoming } + : { ...incoming }; + + const content = mergeLogprobTokens(existing?.content, incoming.content); + if (content !== undefined) { + aggregated.content = content; + } + + const refusal = mergeLogprobTokens(existing?.refusal, incoming.refusal); + if (refusal !== undefined) { + aggregated.refusal = refusal; + } + + return aggregated; +} + +/** + * Aggregate chat completion chunks into a single response. + * Combines role (first), content (concatenated), tool_calls (by id), + * finish_reason (last), and usage (last chunk). + */ +export function aggregateChatCompletionChunks( + chunks: OpenAIChatCompletionChunk[], + streamResult?: unknown, + endEvent?: unknown, +): { + output: OpenAIChatChoice[]; + metrics: Record; +} { + let role = undefined; + let content = undefined; + let refusal = undefined; + let tool_calls = undefined; + let logprobs: OpenAIChatLogprobs | null | undefined = undefined; + let finish_reason = undefined; + let metrics: Record = {}; + + for (const chunk of chunks) { + if (chunk.usage) { + metrics = { + ...metrics, + ...parseMetricsFromUsage(chunk.usage), + }; + } + + const choice = chunk.choices?.[0]; + if (!choice) { + continue; + } + + if (choice.finish_reason) { + finish_reason = choice.finish_reason; + } + + logprobs = aggregateChatLogprobs(logprobs, choice.logprobs); + + const delta = choice.delta; + if (!delta) { + continue; + } + + if (delta.finish_reason) { + finish_reason = delta.finish_reason; + } + + if (!role && delta.role) { + role = delta.role; + } + + if (delta.content) { + content = (content || "") + delta.content; + } + + if (delta.refusal) { + refusal = (refusal || "") + delta.refusal; + } + + if (delta.tool_calls) { + const toolDelta = delta.tool_calls[0]; + if ( + !tool_calls || + (toolDelta.id && tool_calls[tool_calls.length - 1].id !== toolDelta.id) + ) { + tool_calls = [ + ...(tool_calls || []), + { + id: toolDelta.id, + type: toolDelta.type, + function: toolDelta.function, + }, + ]; + } else { + tool_calls[tool_calls.length - 1].function.arguments += + toolDelta.function.arguments; + } + } + } + + metrics = withCachedMetric(metrics, streamResult, endEvent); + + return { + metrics, + output: [ + { + index: 0, + message: { + role, + content, + ...(refusal !== undefined ? { refusal } : {}), + tool_calls, + }, + logprobs: logprobs ?? null, + finish_reason, + }, + ], + }; +} + +function aggregateResponseStreamEvents( + chunks: OpenAIResponseStreamEvent[], + _streamResult?: unknown, + endEvent?: unknown, +): { + output: any; + metrics: Record; + metadata?: Record; +} { + let output: any = undefined; + let metrics: Record = {}; + let metadata: Record | undefined = undefined; + + for (const chunk of chunks) { + if (!chunk || !chunk.type || !chunk.response) { + continue; + } + if (chunk.type !== "response.completed") { + continue; + } + + const response = chunk.response; + if (response?.output !== undefined) { + output = processImagesInOutput(response.output); + } + + const { usage: _usage, output: _output, ...rest } = response || {}; + if (Object.keys(rest).length > 0) { + metadata = rest; + } + + metrics = parseMetricsFromUsage(response?.usage); + } + + return { + output, + metrics: withCachedMetric(metrics, undefined, endEvent), + ...(metadata !== undefined ? { metadata } : {}), + }; +} + +export { parseMetricsFromUsage }; diff --git a/js/src/instrumentation/plugins/openrouter-agent-channels.ts b/js/src/instrumentation/providers/openrouter-agent-channels.ts similarity index 100% rename from js/src/instrumentation/plugins/openrouter-agent-channels.ts rename to js/src/instrumentation/providers/openrouter-agent-channels.ts diff --git a/js/src/instrumentation/plugins/openrouter-agent-plugin.test.ts b/js/src/instrumentation/providers/openrouter-agent-instrumentation.test.ts similarity index 99% rename from js/src/instrumentation/plugins/openrouter-agent-plugin.test.ts rename to js/src/instrumentation/providers/openrouter-agent-instrumentation.test.ts index 85d66d51c..27fa78ca2 100644 --- a/js/src/instrumentation/plugins/openrouter-agent-plugin.test.ts +++ b/js/src/instrumentation/providers/openrouter-agent-instrumentation.test.ts @@ -14,7 +14,7 @@ import { aggregateOpenRouterChatChunks, aggregateOpenRouterResponseStreamEvents, parseOpenRouterMetricsFromUsage, -} from "./openrouter-agent-plugin"; +} from "./openrouter-agent-instrumentation"; const TEST_PROVIDER = "openai"; const TEST_MODEL = "gpt-4.1-mini"; @@ -25,7 +25,7 @@ try { // Best-effort initialization for test environments. } -describe("OpenRouter Agent Plugin", () => { +describe("OpenRouter Agent instrumentation", () => { let backgroundLogger: ReturnType< typeof _exportsForTestingOnly.useTestBackgroundLogger >; @@ -37,7 +37,7 @@ describe("OpenRouter Agent Plugin", () => { beforeEach(() => { backgroundLogger = _exportsForTestingOnly.useTestBackgroundLogger(); initLogger({ - projectName: "openrouter-agent-plugin.test.ts", + projectName: "openrouter-agent-instrumentation.test.ts", projectId: "test-project-id", }); }); diff --git a/js/src/instrumentation/plugins/openrouter-agent-plugin.ts b/js/src/instrumentation/providers/openrouter-agent-instrumentation.ts similarity index 91% rename from js/src/instrumentation/plugins/openrouter-agent-plugin.ts rename to js/src/instrumentation/providers/openrouter-agent-instrumentation.ts index facda63c0..7f77cccfc 100644 --- a/js/src/instrumentation/plugins/openrouter-agent-plugin.ts +++ b/js/src/instrumentation/providers/openrouter-agent-instrumentation.ts @@ -1,9 +1,8 @@ -import { BasePlugin, toLoggedError } from "../core"; +import { toLoggedError } from "../core"; import { traceAsyncChannel, traceStreamingChannel, traceSyncStreamChannel, - unsubscribeAll, } from "../core/channel-tracing"; import type { ChannelMessage } from "../core/channel-definitions"; import { @@ -26,107 +25,93 @@ import type { OpenRouterAgentToolTurnContext, } from "../../vendor-sdk-types/openrouter-agent"; -export class OpenRouterAgentPlugin extends BasePlugin { - protected onEnable(): void { +class OpenRouterAgentInstrumentationConsumer { + public register(): void { this.subscribeToOpenRouterAgentChannels(); } - protected onDisable(): void { - this.unsubscribers = unsubscribeAll(this.unsubscribers); - } - private subscribeToOpenRouterAgentChannels(): void { - this.unsubscribers.push( - traceSyncStreamChannel(openRouterAgentChannels.callModel, { - name: "openrouter.callModel", - type: SpanTypeAttribute.TASK, - extractInput: (args) => { - const request = getOpenRouterCallModelRequestArg(args); - return { - input: request - ? extractOpenRouterCallModelInput(request) - : undefined, - metadata: request - ? extractOpenRouterCallModelMetadata(request) - : { provider: "openrouter" }, - }; - }, - patchResult: ({ endEvent, result, span }) => { - return patchOpenRouterCallModelResult({ - request: getOpenRouterCallModelRequestArg(endEvent.arguments), - result, - span, - }); - }, - }), - ); - - this.unsubscribers.push( - traceAsyncChannel(openRouterAgentChannels.callModelTurn, { - name: "openrouter.beta.responses.send", - type: SpanTypeAttribute.LLM, - extractInput: (args, event) => { - const request = getOpenRouterCallModelRequestArg(args); - const metadata = request + traceSyncStreamChannel(openRouterAgentChannels.callModel, { + name: "openrouter.callModel", + type: SpanTypeAttribute.TASK, + extractInput: (args) => { + const request = getOpenRouterCallModelRequestArg(args); + return { + input: request ? extractOpenRouterCallModelInput(request) : undefined, + metadata: request ? extractOpenRouterCallModelMetadata(request) - : { provider: "openrouter" }; + : { provider: "openrouter" }, + }; + }, + patchResult: ({ endEvent, result, span }) => { + return patchOpenRouterCallModelResult({ + request: getOpenRouterCallModelRequestArg(endEvent.arguments), + result, + span, + }); + }, + }); - if (isObject(metadata) && "tools" in metadata) { - delete (metadata as Record).tools; - } + traceAsyncChannel(openRouterAgentChannels.callModelTurn, { + name: "openrouter.beta.responses.send", + type: SpanTypeAttribute.LLM, + extractInput: (args, event) => { + const request = getOpenRouterCallModelRequestArg(args); + const metadata = request + ? extractOpenRouterCallModelMetadata(request) + : { provider: "openrouter" }; + + if (isObject(metadata) && "tools" in metadata) { + delete (metadata as Record).tools; + } + return { + input: request ? extractOpenRouterCallModelInput(request) : undefined, + metadata: { + ...metadata, + step: event.step, + step_type: event.stepType, + }, + }; + }, + extractOutput: (result) => + extractOpenRouterResponseOutput(result as Record), + extractMetadata: (result, event) => { + if (!isObject(result)) { return { - input: request - ? extractOpenRouterCallModelInput(request) - : undefined, - metadata: { - ...metadata, - step: event.step, - step_type: event.stepType, - }, + step: event?.step, + step_type: event?.stepType, }; - }, - extractOutput: (result) => - extractOpenRouterResponseOutput(result as Record), - extractMetadata: (result, event) => { - if (!isObject(result)) { - return { - step: event?.step, - step_type: event?.stepType, - }; - } + } - return { - ...(extractOpenRouterResponseMetadata(result) || {}), - ...(event?.step !== undefined ? { step: event.step } : {}), - ...(event?.stepType ? { step_type: event.stepType } : {}), - }; + return { + ...(extractOpenRouterResponseMetadata(result) || {}), + ...(event?.step !== undefined ? { step: event.step } : {}), + ...(event?.stepType ? { step_type: event.stepType } : {}), + }; + }, + extractMetrics: (result) => + isObject(result) ? parseOpenRouterMetricsFromUsage(result.usage) : {}, + }); + + traceStreamingChannel(openRouterAgentChannels.toolExecute, { + name: "openrouter.tool", + type: SpanTypeAttribute.TOOL, + extractInput: (args, event) => ({ + input: args[0], + metadata: { + provider: "openrouter", + tool_name: event.toolName, + ...(event.toolCallId ? { tool_call_id: event.toolCallId } : {}), }, - extractMetrics: (result) => - isObject(result) ? parseOpenRouterMetricsFromUsage(result.usage) : {}, }), - ); - - this.unsubscribers.push( - traceStreamingChannel(openRouterAgentChannels.toolExecute, { - name: "openrouter.tool", - type: SpanTypeAttribute.TOOL, - extractInput: (args, event) => ({ - input: args[0], - metadata: { - provider: "openrouter", - tool_name: event.toolName, - ...(event.toolCallId ? { tool_call_id: event.toolCallId } : {}), - }, - }), - extractOutput: (result) => result, - extractMetrics: () => ({}), - aggregateChunks: (chunks) => ({ - output: chunks.length > 0 ? chunks[chunks.length - 1] : undefined, - metrics: {}, - }), + extractOutput: (result) => result, + extractMetrics: () => ({}), + aggregateChunks: (chunks) => ({ + output: chunks.length > 0 ? chunks[chunks.length - 1] : undefined, + metrics: {}, }), - ); + }); const callModelChannel = openRouterAgentChannels.callModel.tracingChannel(); const callModelHandlers = { @@ -141,9 +126,6 @@ export class OpenRouterAgentPlugin extends BasePlugin { }; callModelChannel.subscribe(callModelHandlers); - this.unsubscribers.push(() => { - callModelChannel.unsubscribe(callModelHandlers); - }); } } @@ -1326,3 +1308,13 @@ function normalizeError(error: unknown): Error { } export { parseOpenRouterMetricsFromUsage }; + +let openRouterAgentInstrumentationConsumer: + | OpenRouterAgentInstrumentationConsumer + | undefined; + +export function registerOpenRouterAgentInstrumentation(): void { + openRouterAgentInstrumentationConsumer ??= + new OpenRouterAgentInstrumentationConsumer(); + openRouterAgentInstrumentationConsumer.register(); +} diff --git a/js/src/instrumentation/plugins/openrouter-channels.ts b/js/src/instrumentation/providers/openrouter-channels.ts similarity index 100% rename from js/src/instrumentation/plugins/openrouter-channels.ts rename to js/src/instrumentation/providers/openrouter-channels.ts diff --git a/js/src/instrumentation/plugins/openrouter-plugin.test.ts b/js/src/instrumentation/providers/openrouter-instrumentation.test.ts similarity index 99% rename from js/src/instrumentation/plugins/openrouter-plugin.test.ts rename to js/src/instrumentation/providers/openrouter-instrumentation.test.ts index 93627512e..c656d5e6d 100644 --- a/js/src/instrumentation/plugins/openrouter-plugin.test.ts +++ b/js/src/instrumentation/providers/openrouter-instrumentation.test.ts @@ -14,7 +14,7 @@ import { aggregateOpenRouterChatChunks, aggregateOpenRouterResponseStreamEvents, parseOpenRouterMetricsFromUsage, -} from "./openrouter-plugin"; +} from "./openrouter-instrumentation"; const TEST_PROVIDER = "openai"; const TEST_MODEL = "gpt-4.1-mini"; @@ -27,7 +27,7 @@ try { // Best-effort initialization for test environments. } -describe("OpenRouter Plugin", () => { +describe("OpenRouter instrumentation", () => { let backgroundLogger: ReturnType< typeof _exportsForTestingOnly.useTestBackgroundLogger >; @@ -39,7 +39,7 @@ describe("OpenRouter Plugin", () => { beforeEach(() => { backgroundLogger = _exportsForTestingOnly.useTestBackgroundLogger(); initLogger({ - projectName: "openrouter-plugin.test.ts", + projectName: "openrouter-instrumentation.test.ts", projectId: "test-project-id", }); }); diff --git a/js/src/instrumentation/plugins/openrouter-plugin.ts b/js/src/instrumentation/providers/openrouter-instrumentation.ts similarity index 80% rename from js/src/instrumentation/plugins/openrouter-plugin.ts rename to js/src/instrumentation/providers/openrouter-instrumentation.ts index c89b8f9a7..fa8e54aaa 100644 --- a/js/src/instrumentation/plugins/openrouter-plugin.ts +++ b/js/src/instrumentation/providers/openrouter-instrumentation.ts @@ -1,9 +1,8 @@ -import { BasePlugin, toLoggedError } from "../core"; +import { toLoggedError } from "../core"; import { traceAsyncChannel, traceStreamingChannel, traceSyncStreamChannel, - unsubscribeAll, } from "../core/channel-tracing"; import type { ChannelMessage } from "../core/channel-definitions"; import { @@ -28,247 +27,225 @@ import type { OpenRouterToolTurnContext, } from "../../vendor-sdk-types/openrouter"; -export class OpenRouterPlugin extends BasePlugin { - protected onEnable(): void { +class OpenRouterInstrumentationConsumer { + public register(): void { this.subscribeToOpenRouterChannels(); } - protected onDisable(): void { - this.unsubscribers = unsubscribeAll(this.unsubscribers); - } - private subscribeToOpenRouterChannels(): void { - this.unsubscribers.push( - traceStreamingChannel(openRouterChannels.chatSend, { - name: "openrouter.chat.send", - type: SpanTypeAttribute.LLM, - extractInput: (args) => { - const request = getOpenRouterRequestArg(args); - const chatGenerationParams = isObject(request?.chatGenerationParams) - ? request.chatGenerationParams - : {}; - const httpReferer = request?.httpReferer; - const xTitle = request?.xTitle; - const { messages, ...metadata } = chatGenerationParams; - return { - input: messages, - metadata: buildOpenRouterMetadata(metadata, httpReferer, xTitle), - }; - }, - extractOutput: (result) => { - return isObject(result) ? result.choices : undefined; - }, - extractMetrics: (result, startTime) => { - const metrics = parseOpenRouterMetricsFromUsage(result?.usage); - if (startTime) { - metrics.time_to_first_token = getCurrentUnixTimestamp() - startTime; - } - return metrics; - }, - aggregateChunks: aggregateOpenRouterChatChunks, - }), - ); - - this.unsubscribers.push( - traceAsyncChannel(openRouterChannels.embeddingsGenerate, { - name: "openrouter.embeddings.generate", - type: SpanTypeAttribute.LLM, - extractInput: (args) => { - const request = getOpenRouterRequestArg(args); - const requestBody = isObject(request?.requestBody) - ? request.requestBody - : {}; - const httpReferer = request?.httpReferer; - const xTitle = request?.xTitle; - const { input, ...metadata } = requestBody; - return { - input, - metadata: buildOpenRouterEmbeddingMetadata( - metadata, - httpReferer, - xTitle, - ), - }; - }, - extractOutput: (result) => { - if (!isObject(result)) { - return undefined; - } + traceStreamingChannel(openRouterChannels.chatSend, { + name: "openrouter.chat.send", + type: SpanTypeAttribute.LLM, + extractInput: (args) => { + const request = getOpenRouterRequestArg(args); + const chatGenerationParams = isObject(request?.chatGenerationParams) + ? request.chatGenerationParams + : {}; + const httpReferer = request?.httpReferer; + const xTitle = request?.xTitle; + const { messages, ...metadata } = chatGenerationParams; + return { + input: messages, + metadata: buildOpenRouterMetadata(metadata, httpReferer, xTitle), + }; + }, + extractOutput: (result) => { + return isObject(result) ? result.choices : undefined; + }, + extractMetrics: (result, startTime) => { + const metrics = parseOpenRouterMetricsFromUsage(result?.usage); + if (startTime) { + metrics.time_to_first_token = getCurrentUnixTimestamp() - startTime; + } + return metrics; + }, + aggregateChunks: aggregateOpenRouterChatChunks, + }); - const embedding = result.data?.[0]?.embedding; - return Array.isArray(embedding) - ? { embedding_length: embedding.length } - : undefined; - }, - extractMetadata: (result) => { - if (!isObject(result)) { - return undefined; - } + traceAsyncChannel(openRouterChannels.embeddingsGenerate, { + name: "openrouter.embeddings.generate", + type: SpanTypeAttribute.LLM, + extractInput: (args) => { + const request = getOpenRouterRequestArg(args); + const requestBody = isObject(request?.requestBody) + ? request.requestBody + : {}; + const httpReferer = request?.httpReferer; + const xTitle = request?.xTitle; + const { input, ...metadata } = requestBody; + return { + input, + metadata: buildOpenRouterEmbeddingMetadata( + metadata, + httpReferer, + xTitle, + ), + }; + }, + extractOutput: (result) => { + if (!isObject(result)) { + return undefined; + } - return extractOpenRouterResponseMetadata(result); - }, - extractMetrics: (result) => { - return isObject(result) - ? parseOpenRouterMetricsFromUsage(result.usage) - : {}; - }, - }), - ); + const embedding = result.data?.[0]?.embedding; + return Array.isArray(embedding) + ? { embedding_length: embedding.length } + : undefined; + }, + extractMetadata: (result) => { + if (!isObject(result)) { + return undefined; + } - this.unsubscribers.push( - traceAsyncChannel(openRouterChannels.rerankRerank, { - name: "openrouter.rerank.rerank", - type: SpanTypeAttribute.LLM, - extractInput: (args) => { - const request = getOpenRouterRequestArg(args); - const requestBody = isObject(request?.requestBody) - ? request.requestBody - : {}; - const httpReferer = request?.httpReferer; - const xTitle = request?.xTitle ?? request?.appTitle; - const { documents, query, ...metadata } = requestBody; - return { - input: { - documents, - query, - }, - metadata: buildOpenRouterRerankMetadata( - metadata, - documents, - httpReferer, - xTitle, - ), - }; - }, - extractOutput: (result) => extractOpenRouterRerankOutput(result), - extractMetadata: (result) => extractOpenRouterResponseMetadata(result), - extractMetrics: (result) => - isObject(result) ? parseOpenRouterMetricsFromUsage(result.usage) : {}, - }), - ); + return extractOpenRouterResponseMetadata(result); + }, + extractMetrics: (result) => { + return isObject(result) + ? parseOpenRouterMetricsFromUsage(result.usage) + : {}; + }, + }); - this.unsubscribers.push( - traceStreamingChannel(openRouterChannels.betaResponsesSend, { - name: "openrouter.beta.responses.send", - type: SpanTypeAttribute.LLM, - extractInput: (args) => { - const request = getOpenRouterRequestArg(args); - const openResponsesRequest = isObject(request?.openResponsesRequest) - ? request.openResponsesRequest - : {}; - const httpReferer = request?.httpReferer; - const xTitle = request?.xTitle; - const { input, ...metadata } = openResponsesRequest; - return { - input, - metadata: buildOpenRouterMetadata(metadata, httpReferer, xTitle), - }; - }, - extractOutput: (result) => - extractOpenRouterResponseOutput(result as Record), - extractMetadata: (result) => extractOpenRouterResponseMetadata(result), - extractMetrics: (result, startTime) => { - const metrics = parseOpenRouterMetricsFromUsage(result?.usage); - if (startTime) { - metrics.time_to_first_token = getCurrentUnixTimestamp() - startTime; - } - return metrics; - }, - aggregateChunks: aggregateOpenRouterResponseStreamEvents, - }), - ); + traceAsyncChannel(openRouterChannels.rerankRerank, { + name: "openrouter.rerank.rerank", + type: SpanTypeAttribute.LLM, + extractInput: (args) => { + const request = getOpenRouterRequestArg(args); + const requestBody = isObject(request?.requestBody) + ? request.requestBody + : {}; + const httpReferer = request?.httpReferer; + const xTitle = request?.xTitle ?? request?.appTitle; + const { documents, query, ...metadata } = requestBody; + return { + input: { + documents, + query, + }, + metadata: buildOpenRouterRerankMetadata( + metadata, + documents, + httpReferer, + xTitle, + ), + }; + }, + extractOutput: (result) => extractOpenRouterRerankOutput(result), + extractMetadata: (result) => extractOpenRouterResponseMetadata(result), + extractMetrics: (result) => + isObject(result) ? parseOpenRouterMetricsFromUsage(result.usage) : {}, + }); - this.unsubscribers.push( - traceSyncStreamChannel(openRouterChannels.callModel, { - name: "openrouter.callModel", - type: SpanTypeAttribute.TASK, - extractInput: (args) => { - const request = getOpenRouterCallModelRequestArg(args); - return { - input: request - ? extractOpenRouterCallModelInput(request) - : undefined, - metadata: request - ? extractOpenRouterCallModelMetadata(request) - : { provider: "openrouter" }, - }; - }, - patchResult: ({ endEvent, result, span }) => { - return patchOpenRouterCallModelResult({ - request: getOpenRouterCallModelRequestArg(endEvent.arguments), - result, - span, - }); - }, - }), - ); + traceStreamingChannel(openRouterChannels.betaResponsesSend, { + name: "openrouter.beta.responses.send", + type: SpanTypeAttribute.LLM, + extractInput: (args) => { + const request = getOpenRouterRequestArg(args); + const openResponsesRequest = isObject(request?.openResponsesRequest) + ? request.openResponsesRequest + : {}; + const httpReferer = request?.httpReferer; + const xTitle = request?.xTitle; + const { input, ...metadata } = openResponsesRequest; + return { + input, + metadata: buildOpenRouterMetadata(metadata, httpReferer, xTitle), + }; + }, + extractOutput: (result) => + extractOpenRouterResponseOutput(result as Record), + extractMetadata: (result) => extractOpenRouterResponseMetadata(result), + extractMetrics: (result, startTime) => { + const metrics = parseOpenRouterMetricsFromUsage(result?.usage); + if (startTime) { + metrics.time_to_first_token = getCurrentUnixTimestamp() - startTime; + } + return metrics; + }, + aggregateChunks: aggregateOpenRouterResponseStreamEvents, + }); - this.unsubscribers.push( - traceAsyncChannel(openRouterChannels.callModelTurn, { - name: "openrouter.beta.responses.send", - type: SpanTypeAttribute.LLM, - extractInput: (args, event) => { - const request = getOpenRouterCallModelRequestArg(args); - const metadata = request + traceSyncStreamChannel(openRouterChannels.callModel, { + name: "openrouter.callModel", + type: SpanTypeAttribute.TASK, + extractInput: (args) => { + const request = getOpenRouterCallModelRequestArg(args); + return { + input: request ? extractOpenRouterCallModelInput(request) : undefined, + metadata: request ? extractOpenRouterCallModelMetadata(request) - : { provider: "openrouter" }; + : { provider: "openrouter" }, + }; + }, + patchResult: ({ endEvent, result, span }) => { + return patchOpenRouterCallModelResult({ + request: getOpenRouterCallModelRequestArg(endEvent.arguments), + result, + span, + }); + }, + }); - if (isObject(metadata) && "tools" in metadata) { - delete (metadata as Record).tools; - } + traceAsyncChannel(openRouterChannels.callModelTurn, { + name: "openrouter.beta.responses.send", + type: SpanTypeAttribute.LLM, + extractInput: (args, event) => { + const request = getOpenRouterCallModelRequestArg(args); + const metadata = request + ? extractOpenRouterCallModelMetadata(request) + : { provider: "openrouter" }; + + if (isObject(metadata) && "tools" in metadata) { + delete (metadata as Record).tools; + } + return { + input: request ? extractOpenRouterCallModelInput(request) : undefined, + metadata: { + ...metadata, + step: event.step, + step_type: event.stepType, + }, + }; + }, + extractOutput: (result) => + extractOpenRouterResponseOutput(result as Record), + extractMetadata: (result, event) => { + if (!isObject(result)) { return { - input: request - ? extractOpenRouterCallModelInput(request) - : undefined, - metadata: { - ...metadata, - step: event.step, - step_type: event.stepType, - }, + step: event?.step, + step_type: event?.stepType, }; - }, - extractOutput: (result) => - extractOpenRouterResponseOutput(result as Record), - extractMetadata: (result, event) => { - if (!isObject(result)) { - return { - step: event?.step, - step_type: event?.stepType, - }; - } + } - return { - ...(extractOpenRouterResponseMetadata(result) || {}), - ...(event?.step !== undefined ? { step: event.step } : {}), - ...(event?.stepType ? { step_type: event.stepType } : {}), - }; + return { + ...(extractOpenRouterResponseMetadata(result) || {}), + ...(event?.step !== undefined ? { step: event.step } : {}), + ...(event?.stepType ? { step_type: event.stepType } : {}), + }; + }, + extractMetrics: (result) => + isObject(result) ? parseOpenRouterMetricsFromUsage(result.usage) : {}, + }); + + traceStreamingChannel(openRouterChannels.toolExecute, { + name: "openrouter.tool", + type: SpanTypeAttribute.TOOL, + extractInput: (args, event) => ({ + input: args[0], + metadata: { + provider: "openrouter", + tool_name: event.toolName, + ...(event.toolCallId ? { tool_call_id: event.toolCallId } : {}), }, - extractMetrics: (result) => - isObject(result) ? parseOpenRouterMetricsFromUsage(result.usage) : {}, }), - ); - - this.unsubscribers.push( - traceStreamingChannel(openRouterChannels.toolExecute, { - name: "openrouter.tool", - type: SpanTypeAttribute.TOOL, - extractInput: (args, event) => ({ - input: args[0], - metadata: { - provider: "openrouter", - tool_name: event.toolName, - ...(event.toolCallId ? { tool_call_id: event.toolCallId } : {}), - }, - }), - extractOutput: (result) => result, - extractMetrics: () => ({}), - aggregateChunks: (chunks) => ({ - output: chunks.length > 0 ? chunks[chunks.length - 1] : undefined, - metrics: {}, - }), + extractOutput: (result) => result, + extractMetrics: () => ({}), + aggregateChunks: (chunks) => ({ + output: chunks.length > 0 ? chunks[chunks.length - 1] : undefined, + metrics: {}, }), - ); + }); const callModelChannel = openRouterChannels.callModel.tracingChannel(); const callModelHandlers = { @@ -283,9 +260,6 @@ export class OpenRouterPlugin extends BasePlugin { }; callModelChannel.subscribe(callModelHandlers); - this.unsubscribers.push(() => { - callModelChannel.unsubscribe(callModelHandlers); - }); } } @@ -1508,3 +1482,12 @@ function normalizeError(error: unknown): Error { } export { parseOpenRouterMetricsFromUsage }; + +let openRouterInstrumentationConsumer: + | OpenRouterInstrumentationConsumer + | undefined; + +export function registerOpenRouterInstrumentation(): void { + openRouterInstrumentationConsumer ??= new OpenRouterInstrumentationConsumer(); + openRouterInstrumentationConsumer.register(); +} diff --git a/js/src/instrumentation/plugins/pi-coding-agent-channels.ts b/js/src/instrumentation/providers/pi-coding-agent-channels.ts similarity index 100% rename from js/src/instrumentation/plugins/pi-coding-agent-channels.ts rename to js/src/instrumentation/providers/pi-coding-agent-channels.ts diff --git a/js/src/instrumentation/plugins/pi-coding-agent-plugin.test.ts b/js/src/instrumentation/providers/pi-coding-agent-instrumentation.test.ts similarity index 91% rename from js/src/instrumentation/plugins/pi-coding-agent-plugin.test.ts rename to js/src/instrumentation/providers/pi-coding-agent-instrumentation.test.ts index 433361974..7e7d406ed 100644 --- a/js/src/instrumentation/plugins/pi-coding-agent-plugin.test.ts +++ b/js/src/instrumentation/providers/pi-coding-agent-instrumentation.test.ts @@ -30,7 +30,7 @@ vi.mock("../../logger", () => ({ })); import { isAutoInstrumentationSuppressed } from "../auto-instrumentation-suppression"; -import { PiCodingAgentPlugin } from "./pi-coding-agent-plugin"; +import { registerPiCodingAgentInstrumentation } from "./pi-coding-agent-instrumentation"; const PROMPT_CHANNEL = "orchestrion:@earendil-works/pi-coding-agent:AgentSession.prompt"; @@ -43,12 +43,10 @@ type TestSpan = { name?: string; }; -describe("PiCodingAgentPlugin", () => { - let plugins: PiCodingAgentPlugin[]; +describe("registerPiCodingAgentInstrumentation", () => { let spans: TestSpan[]; beforeEach(() => { - plugins = []; spans = []; interceptorsByName.clear(); mockNewTracingChannel.mockImplementation((name: string) => ({ @@ -74,25 +72,19 @@ describe("PiCodingAgentPlugin", () => { }); afterEach(() => { - for (const plugin of plugins) { - plugin.disable(); - } vi.clearAllMocks(); }); - it("registers and removes the prompt interceptor", () => { - const plugin = enablePlugin(plugins); + it("registers the prompt interceptor for the process lifetime", () => { + registerPiCodingAgentInstrumentation(); expect(interceptorsByName.has(PROMPT_CHANNEL)).toBe(true); expect(mockRemoveInterceptor).not.toHaveBeenCalled(); - - plugin.disable(); - - expect(mockRemoveInterceptor).toHaveBeenCalledTimes(1); }); it("uses interceptor ALS for prompt, LLM, and tool spans", async () => { - const interceptor = promptInterceptor(enablePlugin(plugins)); + registerPiCodingAgentInstrumentation(); + const interceptor = promptInterceptor(); const finalMessage = makeAssistantMessage("done"); const originalStreamFn = vi.fn(async () => { expect(isAutoInstrumentationSuppressed()).toBe(true); @@ -200,7 +192,8 @@ describe("PiCodingAgentPlugin", () => { }); it("isolates overlapping prompts on the same agent without prompt matching", async () => { - const interceptor = promptInterceptor(enablePlugin(plugins)); + registerPiCodingAgentInstrumentation(); + const interceptor = promptInterceptor(); const originalStreamFn = vi.fn(async () => makeStream(makeAssistantMessage("done")), ); @@ -258,7 +251,8 @@ describe("PiCodingAgentPlugin", () => { }); it("keeps deferred follow-up prompts open for their ALS-owned turn", async () => { - const interceptor = promptInterceptor(enablePlugin(plugins)); + registerPiCodingAgentInstrumentation(); + const interceptor = promptInterceptor(); const agent = makeAgent( vi.fn(async () => makeStream(makeAssistantMessage("done"))), ); @@ -296,7 +290,8 @@ describe("PiCodingAgentPlugin", () => { }); it("preserves full-iterator stream behavior", async () => { - const interceptor = promptInterceptor(enablePlugin(plugins)); + registerPiCodingAgentInstrumentation(); + const interceptor = promptInterceptor(); const message = makeAssistantMessage("done"); const { result, stream } = makeIteratorBackedStream([ { partial: message, type: "start" }, @@ -331,7 +326,8 @@ describe("PiCodingAgentPlugin", () => { }); it("forwards iterator failures and closes the underlying iterator", async () => { - const interceptor = promptInterceptor(enablePlugin(plugins)); + registerPiCodingAgentInstrumentation(); + const interceptor = promptInterceptor(); const { iterator, stream } = makeIteratorBackedStream([]); iterator.next.mockRejectedValueOnce(new Error("stream next failed")); const agent = makeAgent(vi.fn(async () => stream)); @@ -364,7 +360,8 @@ describe("PiCodingAgentPlugin", () => { }); it("closes active prompt and LLM spans when the target rejects", async () => { - const interceptor = promptInterceptor(enablePlugin(plugins)); + registerPiCodingAgentInstrumentation(); + const interceptor = promptInterceptor(); const agent = makeAgent( vi.fn(async () => makeStream(makeAssistantMessage("unused"))), ); @@ -396,9 +393,9 @@ describe("PiCodingAgentPlugin", () => { expect(taskSpan?.end).toHaveBeenCalledTimes(1); }); - it("ends in-flight prompts on disable while leaving the safe patch installed", async () => { - const plugin = enablePlugin(plugins); - const interceptor = promptInterceptor(plugin); + it("keeps in-flight prompts active until the call completes", async () => { + registerPiCodingAgentInstrumentation(); + const interceptor = promptInterceptor(); const originalStreamFn = vi.fn(async () => makeStream(makeAssistantMessage("done")), ); @@ -416,12 +413,10 @@ describe("PiCodingAgentPlugin", () => { ); const patchedStreamFn = agent.streamFn; - plugin.disable(); - const taskSpan = findSpan(spans, "AgentSession.prompt"); expect(agent.streamFn).toBe(patchedStreamFn); expect(agent.streamFn).not.toBe(originalStreamFn); - expect(taskSpan?.end).toHaveBeenCalledTimes(1); + expect(taskSpan?.end).not.toHaveBeenCalled(); targetGate.resolve(); await result; @@ -429,14 +424,7 @@ describe("PiCodingAgentPlugin", () => { }); }); -function enablePlugin(plugins: PiCodingAgentPlugin[]): PiCodingAgentPlugin { - const plugin = new PiCodingAgentPlugin(); - plugins.push(plugin); - plugin.enable(); - return plugin; -} - -function promptInterceptor(_plugin: PiCodingAgentPlugin): any { +function promptInterceptor(): any { const interceptor = interceptorsByName.get(PROMPT_CHANNEL); expect(interceptor).toEqual(expect.any(Function)); return interceptor; diff --git a/js/src/instrumentation/plugins/pi-coding-agent-plugin.ts b/js/src/instrumentation/providers/pi-coding-agent-instrumentation.ts similarity index 93% rename from js/src/instrumentation/plugins/pi-coding-agent-plugin.ts rename to js/src/instrumentation/providers/pi-coding-agent-instrumentation.ts index da6c50b30..11829e004 100644 --- a/js/src/instrumentation/plugins/pi-coding-agent-plugin.ts +++ b/js/src/instrumentation/providers/pi-coding-agent-instrumentation.ts @@ -1,4 +1,4 @@ -import { BasePlugin, toLoggedError } from "../core"; +import { toLoggedError } from "../core"; import type { ChannelMessage } from "../core/channel-definitions"; import iso, { type IsoAsyncLocalStorage } from "../../isomorph"; import { debugLogger } from "../../debug-logger"; @@ -44,7 +44,6 @@ type PiPromptState = { deferCompletionUntilTurnEnd: boolean; finalized: boolean; metrics: Record; - onFinalize?: (state: PiPromptState) => void; metadata: Record; output?: unknown; promptCallEnded: boolean; @@ -77,74 +76,52 @@ let piPromptContextStore: | IsoAsyncLocalStorage | undefined; -export class PiCodingAgentPlugin extends BasePlugin { - private readonly activePromptStates = new Set(); - - protected onEnable(): void { +class PiCodingAgentInstrumentationConsumer { + public register(): void { this.interceptPrompt(); } - protected onDisable(): void { - for (const unsubscribe of this.unsubscribers) { - unsubscribe(); - } - this.unsubscribers = []; - - for (const state of [...this.activePromptStates]) { - finishPiPromptRun(state); - } - } - private interceptPrompt(): void { - this.unsubscribers.push( - piCodingAgentChannels.prompt.intercept( - (target, thisArg, args, additional) => { - const invokeTarget = () => Reflect.apply(target, thisArg, args); - let state: PiPromptState | undefined; - try { - state = startPiPromptRun( - { - ...additional, - arguments: args, - self: thisArg, - }, - (finalizedState) => { - this.activePromptStates.delete(finalizedState); - }, - ); - } catch (error) { - logInstrumentationError("Pi Coding Agent prompt start", error); - } + void piCodingAgentChannels.prompt.intercept( + (target, thisArg, args, additional) => { + const invokeTarget = () => Reflect.apply(target, thisArg, args); + let state: PiPromptState | undefined; + try { + state = startPiPromptRun({ + ...additional, + arguments: args, + self: thisArg, + }); + } catch (error) { + logInstrumentationError("Pi Coding Agent prompt start", error); + } - if (!state) { - return runWithAutoInstrumentationSuppressed(invokeTarget); - } - this.activePromptStates.add(state); + if (!state) { + return runWithAutoInstrumentationSuppressed(invokeTarget); + } + return promptContextStore().run(state, () => + runWithAutoInstrumentationSuppressed(() => { + let result: PromiseLike; + try { + result = invokeTarget(); + } catch (error) { + finishPiPromptRun(state, error); + throw error; + } - return promptContextStore().run(state, () => - runWithAutoInstrumentationSuppressed(() => { - let result: PromiseLike; - try { - result = invokeTarget(); - } catch (error) { + return Promise.resolve(result).then( + (value) => { + finishPiPromptCall(state); + return value; + }, + (error) => { finishPiPromptRun(state, error); throw error; - } - - return Promise.resolve(result).then( - (value) => { - finishPiPromptCall(state); - return value; - }, - (error) => { - finishPiPromptRun(state, error); - throw error; - }, - ); - }), - ); - }, - ), + }, + ); + }), + ); + }, ); } } @@ -159,7 +136,6 @@ function finishPiPromptCall(state: PiPromptState): void { function startPiPromptRun( event: ChannelMessage, - onFinalize?: (state: PiPromptState) => void, ): PiPromptState | undefined { const session = extractSession(event); const agent = session?.agent; @@ -206,7 +182,6 @@ function startPiPromptRun( finalized: false, metadata, metrics: {}, - onFinalize, promptCallEnded: false, span, startTime: getCurrentUnixTimestamp(), @@ -595,7 +570,6 @@ function finishPiPromptRun(state: PiPromptState, error?: unknown): void { return; } state.finalized = true; - state.onFinalize?.(state); finishOpenLlmSpans(state, error); finishOpenToolSpans(state, error); @@ -1085,3 +1059,13 @@ function safeLog(span: Span, event: Parameters[0]): void { function logInstrumentationError(context: string, error: unknown): void { debugLogger.debug(`${context}:`, error); } + +let piCodingAgentInstrumentationConsumer: + | PiCodingAgentInstrumentationConsumer + | undefined; + +export function registerPiCodingAgentInstrumentation(): void { + piCodingAgentInstrumentationConsumer ??= + new PiCodingAgentInstrumentationConsumer(); + piCodingAgentInstrumentationConsumer.register(); +} diff --git a/js/src/instrumentation/plugins/strands-agent-sdk-channels.ts b/js/src/instrumentation/providers/strands-agent-sdk-channels.ts similarity index 100% rename from js/src/instrumentation/plugins/strands-agent-sdk-channels.ts rename to js/src/instrumentation/providers/strands-agent-sdk-channels.ts diff --git a/js/src/instrumentation/plugins/strands-agent-sdk-plugin.test.ts b/js/src/instrumentation/providers/strands-agent-sdk-instrumentation.test.ts similarity index 95% rename from js/src/instrumentation/plugins/strands-agent-sdk-plugin.test.ts rename to js/src/instrumentation/providers/strands-agent-sdk-instrumentation.test.ts index 54340d1a3..5c0986ca8 100644 --- a/js/src/instrumentation/plugins/strands-agent-sdk-plugin.test.ts +++ b/js/src/instrumentation/providers/strands-agent-sdk-instrumentation.test.ts @@ -5,7 +5,6 @@ const { mockWithCurrent, mockNewAsyncLocalStorage, mockStartSpan, - mockUnbindStore, } = vi.hoisted(() => ({ mockBindStore: vi.fn(), mockWithCurrent: vi.fn(), @@ -28,7 +27,6 @@ const { }; }), mockStartSpan: vi.fn(), - mockUnbindStore: vi.fn(), })); vi.mock("../../isomorph", () => ({ @@ -51,11 +49,11 @@ vi.mock("../../logger", async (importOriginal) => { import iso from "../../isomorph"; import { Attachment } from "../../logger"; import { isAutoInstrumentationSuppressed } from "../auto-instrumentation-suppression"; -import { StrandsAgentSDKPlugin } from "./strands-agent-sdk-plugin"; +import { registerStrandsAgentSDKInstrumentation } from "./strands-agent-sdk-instrumentation"; const mockNewTracingChannel = iso.newTracingChannel as ReturnType; -describe("StrandsAgentSDKPlugin", () => { +describe("registerStrandsAgentSDKInstrumentation", () => { let handlersByName: Map; let spans: Array<{ args: any; @@ -77,11 +75,9 @@ describe("StrandsAgentSDKPlugin", () => { mockNewTracingChannel.mockImplementation((name: string) => ({ start: { bindStore: mockBindStore, - unbindStore: mockUnbindStore, }, subscribe: vi.fn((handlers) => handlersByName.set(name, handlers)), traceSync: vi.fn((fn) => fn()), - unsubscribe: vi.fn(), })); currentSpan = undefined; mockWithCurrent.mockImplementation((span: any, callback: () => unknown) => { @@ -123,8 +119,7 @@ describe("StrandsAgentSDKPlugin", () => { }); it("subscribes to Strands stream channels and binds suppression", () => { - const plugin = new StrandsAgentSDKPlugin(); - plugin.enable(); + registerStrandsAgentSDKInstrumentation(); expect( handlersByName.has("orchestrion:@strands-agents/sdk:Agent.stream"), @@ -136,15 +131,10 @@ describe("StrandsAgentSDKPlugin", () => { handlersByName.has("orchestrion:@strands-agents/sdk:Swarm.stream"), ).toBe(true); expect(mockBindStore).toHaveBeenCalledTimes(3); - - plugin.disable(); - - expect(mockUnbindStore).toHaveBeenCalledTimes(3); }); it("records agent model and tool spans from stream events", async () => { - const plugin = new StrandsAgentSDKPlugin(); - plugin.enable(); + registerStrandsAgentSDKInstrumentation(); const handlers = handlersByName.get( "orchestrion:@strands-agents/sdk:Agent.stream", @@ -292,8 +282,7 @@ describe("StrandsAgentSDKPlugin", () => { ])( "converts media from %s to one attachment shared by agent and model spans", async (_description, bytes) => { - const plugin = new StrandsAgentSDKPlugin(); - plugin.enable(); + registerStrandsAgentSDKInstrumentation(); const handlers = handlersByName.get( "orchestrion:@strands-agents/sdk:Agent.stream", @@ -353,8 +342,7 @@ describe("StrandsAgentSDKPlugin", () => { ); it("parents nested agent spans under active graph nodes", async () => { - const plugin = new StrandsAgentSDKPlugin(); - plugin.enable(); + registerStrandsAgentSDKInstrumentation(); const graphHandlers = handlersByName.get( "orchestrion:@strands-agents/sdk:Graph.stream", @@ -441,8 +429,7 @@ describe("StrandsAgentSDKPlugin", () => { }); it("parents nested graph spans under active graph nodes", async () => { - const plugin = new StrandsAgentSDKPlugin(); - plugin.enable(); + registerStrandsAgentSDKInstrumentation(); const graphHandlers = handlersByName.get( "orchestrion:@strands-agents/sdk:Graph.stream", @@ -530,8 +517,7 @@ describe("StrandsAgentSDKPlugin", () => { }); it("ends open child spans when a stream errors", async () => { - const plugin = new StrandsAgentSDKPlugin(); - plugin.enable(); + registerStrandsAgentSDKInstrumentation(); const handlers = handlersByName.get( "orchestrion:@strands-agents/sdk:Agent.stream", diff --git a/js/src/instrumentation/plugins/strands-agent-sdk-plugin.ts b/js/src/instrumentation/providers/strands-agent-sdk-instrumentation.ts similarity index 97% rename from js/src/instrumentation/plugins/strands-agent-sdk-plugin.ts rename to js/src/instrumentation/providers/strands-agent-sdk-instrumentation.ts index 255578a13..5142f2d5c 100644 --- a/js/src/instrumentation/plugins/strands-agent-sdk-plugin.ts +++ b/js/src/instrumentation/providers/strands-agent-sdk-instrumentation.ts @@ -1,4 +1,4 @@ -import { BasePlugin, toLoggedError } from "../core"; +import { toLoggedError } from "../core"; import type { ChannelMessage } from "../core/channel-definitions"; import { isAsyncIterable, patchStreamIfNeeded } from "../core/stream-patcher"; import type { IsoChannelHandlers } from "../../isomorph"; @@ -101,10 +101,10 @@ type StrandsAttachmentCache = { strings: LRUCache>; }; -export class StrandsAgentSDKPlugin extends BasePlugin { +class StrandsAgentSDKInstrumentationConsumer { private readonly activeChildParents: ActiveChildParents = new WeakMap(); - protected onEnable(): void { + public register(): void { this.subscribeToAgentStream(); this.subscribeToMultiAgentStream( strandsAgentSDKChannels.graphStream, @@ -116,18 +116,10 @@ export class StrandsAgentSDKPlugin extends BasePlugin { ); } - protected onDisable(): void { - for (const unsubscribe of this.unsubscribers) { - unsubscribe(); - } - this.unsubscribers = []; - } - private subscribeToAgentStream(): void { const channel = strandsAgentSDKChannels.agentStream.tracingChannel(); const states = new WeakMap(); - const unbindAutoInstrumentationSuppression = - bindAutoInstrumentationSuppressionToStart(channel); + bindAutoInstrumentationSuppressionToStart(channel); const handlers: IsoChannelHandlers< ChannelMessage @@ -176,10 +168,6 @@ export class StrandsAgentSDKPlugin extends BasePlugin { }; channel.subscribe(handlers); - this.unsubscribers.push(() => { - unbindAutoInstrumentationSuppression?.(); - channel.unsubscribe(handlers); - }); } private subscribeToMultiAgentStream( @@ -188,8 +176,7 @@ export class StrandsAgentSDKPlugin extends BasePlugin { ): void { const tracingChannel = channel.tracingChannel(); const states = new WeakMap(); - const unbindAutoInstrumentationSuppression = - bindAutoInstrumentationSuppressionToStart(tracingChannel); + bindAutoInstrumentationSuppressionToStart(tracingChannel); const handlers: IsoChannelHandlers> = { start: (event) => { @@ -250,10 +237,6 @@ export class StrandsAgentSDKPlugin extends BasePlugin { }; tracingChannel.subscribe(handlers); - this.unsubscribers.push(() => { - unbindAutoInstrumentationSuppression?.(); - tracingChannel.unsubscribe(handlers); - }); } } @@ -1393,3 +1376,13 @@ function safeLog(span: Span, event: Parameters[0]): void { function logInstrumentationError(context: string, error: unknown): void { debugLogger.debug(`${context}:`, error); } + +let strandsAgentSDKInstrumentationConsumer: + | StrandsAgentSDKInstrumentationConsumer + | undefined; + +export function registerStrandsAgentSDKInstrumentation(): void { + strandsAgentSDKInstrumentationConsumer ??= + new StrandsAgentSDKInstrumentationConsumer(); + strandsAgentSDKInstrumentationConsumer.register(); +} diff --git a/js/src/instrumentation/plugins/voyageai-channels.ts b/js/src/instrumentation/providers/voyageai-channels.ts similarity index 100% rename from js/src/instrumentation/plugins/voyageai-channels.ts rename to js/src/instrumentation/providers/voyageai-channels.ts diff --git a/js/src/instrumentation/plugins/voyageai-plugin.test.ts b/js/src/instrumentation/providers/voyageai-instrumentation.test.ts similarity index 98% rename from js/src/instrumentation/plugins/voyageai-plugin.test.ts rename to js/src/instrumentation/providers/voyageai-instrumentation.test.ts index 65cfca93c..2cdb491b4 100644 --- a/js/src/instrumentation/plugins/voyageai-plugin.test.ts +++ b/js/src/instrumentation/providers/voyageai-instrumentation.test.ts @@ -9,7 +9,7 @@ try { // Best-effort initialization for test environments. } -describe("VoyageAIPlugin", () => { +describe("registerVoyageAIInstrumentation", () => { let backgroundLogger: ReturnType< typeof _exportsForTestingOnly.useTestBackgroundLogger >; @@ -21,7 +21,7 @@ describe("VoyageAIPlugin", () => { beforeEach(() => { backgroundLogger = _exportsForTestingOnly.useTestBackgroundLogger(); initLogger({ - projectName: "voyageai-plugin.test.ts", + projectName: "voyageai-instrumentation.test.ts", projectId: "test-project-id", }); }); diff --git a/js/src/instrumentation/plugins/voyageai-plugin.ts b/js/src/instrumentation/providers/voyageai-instrumentation.ts similarity index 89% rename from js/src/instrumentation/plugins/voyageai-plugin.ts rename to js/src/instrumentation/providers/voyageai-instrumentation.ts index 37f864b26..ac2304e81 100644 --- a/js/src/instrumentation/plugins/voyageai-plugin.ts +++ b/js/src/instrumentation/providers/voyageai-instrumentation.ts @@ -17,8 +17,6 @@ import { isAutoInstrumentationSuppressed, runWithAutoInstrumentationSuppressed, } from "../auto-instrumentation-suppression"; -import { BasePlugin } from "../core"; -import { unsubscribeAll } from "../core/channel-tracing"; import { voyageAIChannels } from "./voyageai-channels"; const RERANK_METADATA_ALLOWLIST = new Set([ @@ -28,42 +26,34 @@ const RERANK_METADATA_ALLOWLIST = new Set([ "truncation", ]); -export class VoyageAIPlugin extends BasePlugin { - protected onEnable(): void { - this.unsubscribers.push( - interceptVoyageAICall( - voyageAIChannels.embed, - "voyageai.embed", - extractTextEmbeddingInput, - summarizeEmbeddingOutput, - extractEmbeddingUsageMetrics, - ), - interceptVoyageAICall( - voyageAIChannels.multimodalEmbed, - "voyageai.multimodalEmbed", - extractMultimodalEmbeddingInput, - summarizeEmbeddingOutput, - extractEmbeddingUsageMetrics, - ), - interceptVoyageAICall( - voyageAIChannels.rerank, - "voyageai.rerank", - extractRerankInput, - summarizeRerankOutput, - ), - interceptVoyageAICall( - voyageAIChannels.contextualizedEmbed, - "voyageai.contextualizedEmbed", - extractContextualizedEmbeddingInput, - summarizeContextualizedEmbeddingOutput, - extractEmbeddingUsageMetrics, - ), - ); - } - - protected onDisable(): void { - this.unsubscribers = unsubscribeAll(this.unsubscribers); - } +export function registerVoyageAIInstrumentation(): void { + interceptVoyageAICall( + voyageAIChannels.embed, + "voyageai.embed", + extractTextEmbeddingInput, + summarizeEmbeddingOutput, + extractEmbeddingUsageMetrics, + ); + interceptVoyageAICall( + voyageAIChannels.multimodalEmbed, + "voyageai.multimodalEmbed", + extractMultimodalEmbeddingInput, + summarizeEmbeddingOutput, + extractEmbeddingUsageMetrics, + ); + interceptVoyageAICall( + voyageAIChannels.rerank, + "voyageai.rerank", + extractRerankInput, + summarizeRerankOutput, + ); + interceptVoyageAICall( + voyageAIChannels.contextualizedEmbed, + "voyageai.contextualizedEmbed", + extractContextualizedEmbeddingInput, + summarizeContextualizedEmbeddingOutput, + extractEmbeddingUsageMetrics, + ); } type VoyageAIResult = @@ -455,13 +445,10 @@ function extractUsageMetrics(result: VoyageAIResult): Record { const rawResponse = isObject(result.rawResponse) ? result.rawResponse : undefined; - const usage = ( - isObject(result.usage) - ? result.usage - : isObject(rawResponse?.usage) - ? rawResponse.usage - : undefined - ) as VoyageAIUsage | undefined; + const usageValue = isObject(result.usage) ? result.usage : rawResponse?.usage; + const usage = isObject(usageValue) + ? (usageValue as VoyageAIUsage) + : undefined; const tokens = typeof result.totalTokens === "number" ? result.totalTokens diff --git a/js/src/instrumentation/registry.test.ts b/js/src/instrumentation/registry.test.ts index fdd49a5c2..97d325cea 100644 --- a/js/src/instrumentation/registry.test.ts +++ b/js/src/instrumentation/registry.test.ts @@ -19,32 +19,25 @@ import iso from "../isomorph"; const mockNewTracingChannel = iso.newTracingChannel as ReturnType; -describe("Plugin Registry", () => { +describe("Instrumentation Registry", () => { beforeEach(() => { // Setup mock channel const mockChannel = { subscribe: vi.fn(), - unsubscribe: vi.fn(), hasSubscribers: false, }; mockNewTracingChannel.mockReturnValue(mockChannel); }); - // Clean up after each test - afterEach(() => { - registry.disable(); - }); - it("should not be enabled by default", () => { const testRegistry = new (registry.constructor as any)(); expect(testRegistry.isEnabled()).toBe(false); }); - it("should enable plugins when enable() is called", () => { + it("should enable instrumentation consumers when enable() is called", () => { const testRegistry = new (registry.constructor as any)(); testRegistry.enable(); expect(testRegistry.isEnabled()).toBe(true); - testRegistry.disable(); }); it("should be idempotent (calling enable() multiple times)", () => { @@ -52,12 +45,11 @@ describe("Plugin Registry", () => { testRegistry.enable(); testRegistry.enable(); // Should not throw expect(testRegistry.isEnabled()).toBe(true); - testRegistry.disable(); }); it("should block a second instance from subscribing when another is already enabled", () => { // Regression test for BT-5139: when the SDK is loaded from two different - // module paths in the same process, each gets its own PluginRegistry + // module paths in the same process, each gets its own instrumentation registry // instance. Without cross-instance deduplication, both would subscribe to // the same global hook, causing every OpenAI call to produce two // LLM spans. @@ -80,8 +72,6 @@ describe("Plugin Registry", () => { instanceB.enable(); expect(instanceB.isEnabled()).toBe(false); } finally { - instanceA.disable(); - instanceB.disable(); delete (globalThis as any)[stateKey]; } }); @@ -97,7 +87,6 @@ describe("Plugin Registry", () => { testRegistry.enable(); expect(testRegistry.isEnabled()).toBe(true); } finally { - testRegistry.disable(); delete (globalThis as any)[stateKey]; } }); @@ -116,7 +105,6 @@ describe("Plugin Registry", () => { expect(warnSpy[0]).toContain("Cannot configure instrumentation"); } finally { console.warn = originalWarn; - testRegistry.disable(); } }); @@ -133,33 +121,11 @@ describe("Plugin Registry", () => { expect(warnSpy.length).toBe(0); } finally { console.warn = originalWarn; - testRegistry.disable(); } }); - - it("should disable plugins when disable() is called", () => { - const testRegistry = new (registry.constructor as any)(); - testRegistry.enable(); - expect(testRegistry.isEnabled()).toBe(true); - - testRegistry.disable(); - expect(testRegistry.isEnabled()).toBe(false); - }); - - it("should be idempotent (calling disable() multiple times)", () => { - const testRegistry = new (registry.constructor as any)(); - testRegistry.enable(); - testRegistry.disable(); - testRegistry.disable(); // Should not throw - expect(testRegistry.isEnabled()).toBe(false); - }); }); describe("configureInstrumentation API", () => { - afterEach(() => { - registry.disable(); - }); - it("should export configureInstrumentation function", () => { expect(typeof configureInstrumentation).toBe("function"); }); @@ -195,7 +161,6 @@ describe("Environment Variable Configuration", () => { afterEach(async () => { const iso = (await import("../isomorph")).default; iso.getEnv = originalGetEnv; - registry.disable(); }); it("should parse BRAINTRUST_DISABLE_INSTRUMENTATION with single SDK", async () => { @@ -212,7 +177,6 @@ describe("Environment Variable Configuration", () => { // OpenAI should be disabled, others enabled by default expect(testRegistry.isEnabled()).toBe(true); - testRegistry.disable(); }); it("should parse BRAINTRUST_DISABLE_INSTRUMENTATION with multiple SDKs", async () => { @@ -229,7 +193,6 @@ describe("Environment Variable Configuration", () => { // Both should be disabled expect(testRegistry.isEnabled()).toBe(true); - testRegistry.disable(); }); it("should handle whitespace in BRAINTRUST_DISABLE_INSTRUMENTATION", async () => { @@ -245,7 +208,6 @@ describe("Environment Variable Configuration", () => { testRegistry.enable(); expect(testRegistry.isEnabled()).toBe(true); - testRegistry.disable(); }); it("should handle empty BRAINTRUST_DISABLE_INSTRUMENTATION", async () => { @@ -262,7 +224,6 @@ describe("Environment Variable Configuration", () => { // All should be enabled (nothing disabled) expect(testRegistry.isEnabled()).toBe(true); - testRegistry.disable(); }); it("should be case-insensitive for SDK names", async () => { @@ -278,6 +239,5 @@ describe("Environment Variable Configuration", () => { testRegistry.enable(); expect(testRegistry.isEnabled()).toBe(true); - testRegistry.disable(); }); }); diff --git a/js/src/instrumentation/registry.ts b/js/src/instrumentation/registry.ts index 0ed195ae8..010aee634 100644 --- a/js/src/instrumentation/registry.ts +++ b/js/src/instrumentation/registry.ts @@ -1,11 +1,11 @@ /** - * Plugin registry and configuration for auto-instrumentation. + * Registry and configuration for auto-instrumentation. * - * Plugins are automatically enabled when the Braintrust library is loaded. + * Instrumentation consumers are automatically enabled when Braintrust loads. * Users can disable specific integrations programmatically or via environment variables. */ -import { BraintrustPlugin } from "./braintrust-plugin"; +import { registerInstrumentationConsumers } from "./instrumentation-consumers"; import iso from "../isomorph"; import { getDefaultInstrumentationIntegrations, @@ -16,7 +16,7 @@ import { GLOBAL_INSTRUMENTATION_HOOKS_PROTOCOL_VERSION } from "../global-instrum export type { InstrumentationConfig } from "./config"; -// Key used to stamp the active PluginRegistry instance onto the shared +// Key used to stamp the active InstrumentationRegistry instance onto the shared // braintrust state object (globalThis[Symbol.for("braintrust-state")]). // // The braintrust state is already shared across all SDK instances loaded in @@ -43,8 +43,7 @@ function getSharedState(): Record | undefined { : undefined; } -class PluginRegistry { - private braintrustPlugin: BraintrustPlugin | null = null; +class InstrumentationRegistry { private config: InstrumentationConfig = {}; private enabled = false; @@ -65,7 +64,7 @@ class PluginRegistry { } /** - * Enable all configured plugins. + * Enable all configured instrumentation consumers. * Called automatically when the library is loaded. */ enable(): void { @@ -73,7 +72,7 @@ class PluginRegistry { return; } - // If another SDK instance in the same process already registered plugins, + // If another SDK instance already registered instrumentation consumers, // skip to avoid duplicate global hook subscriptions. const sharedState = getSharedState(); if (sharedState) { @@ -95,31 +94,8 @@ class PluginRegistry { }, }; - // Enable BraintrustPlugin with configuration - this.braintrustPlugin = new BraintrustPlugin(finalConfig); - this.braintrustPlugin.enable(); - } - - /** - * Disable all plugins. - * Primarily used for testing. - */ - disable(): void { - if (!this.enabled) { - return; - } - - this.enabled = false; - - const sharedState = getSharedState(); - if (sharedState && sharedState[REGISTRY_STATE_KEY] === this) { - delete sharedState[REGISTRY_STATE_KEY]; - } - - if (this.braintrustPlugin) { - this.braintrustPlugin.disable(); - this.braintrustPlugin = null; - } + // Enable the configured instrumentation consumers. + registerInstrumentationConsumers(finalConfig); } /** @@ -148,9 +124,9 @@ class PluginRegistry { } /** - * Global plugin registry instance. + * Global instrumentation registry instance. */ -export const registry = new PluginRegistry(); +export const registry = new InstrumentationRegistry(); /** * Configure auto-instrumentation. diff --git a/js/src/wrappers/ai-sdk/ai-sdk.ts b/js/src/wrappers/ai-sdk/ai-sdk.ts index 63de58ea6..dc6c7b09d 100644 --- a/js/src/wrappers/ai-sdk/ai-sdk.ts +++ b/js/src/wrappers/ai-sdk/ai-sdk.ts @@ -4,7 +4,7 @@ import { SpanTypeAttribute } from "../../../util"; import { aiSDKChannels, harnessAgentChannels, -} from "../../instrumentation/plugins/ai-sdk-channels"; +} from "../../instrumentation/providers/ai-sdk-channels"; import type { AISDK, AISDKAgentClass, diff --git a/js/src/wrappers/ai-sdk/harness-agent-context.ts b/js/src/wrappers/ai-sdk/harness-agent-context.ts index fc0b6171a..f17f2a442 100644 --- a/js/src/wrappers/ai-sdk/harness-agent-context.ts +++ b/js/src/wrappers/ai-sdk/harness-agent-context.ts @@ -335,10 +335,10 @@ export function currentHarnessTurnParent(): HarnessTurnParent | undefined { export function bindHarnessTurnParentToStart( tracingChannel: IsoTracingChannel, parentFromEvent: (event: T) => HarnessTurnParent | undefined, -): () => void { +): void { const startChannel = tracingChannel.start; if (!startChannel) { - return () => {}; + return; } harnessTurnParentStore ??= iso.newAsyncLocalStorage< @@ -349,9 +349,6 @@ export function bindHarnessTurnParentToStart( store, (event) => parentFromEvent(event) ?? store.getStore(), ); - return () => { - startChannel.unbindStore(store); - }; } export function startHarnessTurnChildSpan( diff --git a/js/src/wrappers/ai-sdk/telemetry.ts b/js/src/wrappers/ai-sdk/telemetry.ts index 1fb2c9812..110f8e221 100644 --- a/js/src/wrappers/ai-sdk/telemetry.ts +++ b/js/src/wrappers/ai-sdk/telemetry.ts @@ -20,7 +20,7 @@ import { processAISDKWorkflowAgentCallInput, processAISDKWorkflowAgentModelCallInput, serializeModelWithProvider, -} from "../../instrumentation/plugins/ai-sdk-plugin"; +} from "../../instrumentation/providers/ai-sdk-instrumentation"; import type { AISDKCallParams, AISDKEmbeddingResult, diff --git a/js/src/wrappers/anthropic.ts b/js/src/wrappers/anthropic.ts index 12aae0218..e4cfc708f 100644 --- a/js/src/wrappers/anthropic.ts +++ b/js/src/wrappers/anthropic.ts @@ -1,4 +1,4 @@ -import { anthropicChannels } from "../instrumentation/plugins/anthropic-channels"; +import { anthropicChannels } from "../instrumentation/providers/anthropic-channels"; import { TypedApplyProxy } from "../typed-instrumentation-helpers"; import type { AnthropicBeta, diff --git a/js/src/wrappers/bedrock-runtime.ts b/js/src/wrappers/bedrock-runtime.ts index 13fcd1844..4e60fd912 100644 --- a/js/src/wrappers/bedrock-runtime.ts +++ b/js/src/wrappers/bedrock-runtime.ts @@ -1,9 +1,9 @@ import { runWithAutoInstrumentationSuppressed } from "../instrumentation/auto-instrumentation-suppression"; -import { bedrockRuntimeChannels } from "../instrumentation/plugins/bedrock-runtime-channels"; +import { bedrockRuntimeChannels } from "../instrumentation/providers/bedrock-runtime-channels"; import { buildBedrockRuntimeSpanInfo, getBedrockRuntimeOperation, -} from "../instrumentation/plugins/bedrock-runtime-common"; +} from "../instrumentation/providers/bedrock-runtime-common"; import type { BedrockRuntimeClient, BedrockRuntimeCommandLike, diff --git a/js/src/wrappers/claude-agent-sdk/claude-agent-sdk.ts b/js/src/wrappers/claude-agent-sdk/claude-agent-sdk.ts index db24f0bcf..fae63e64d 100644 --- a/js/src/wrappers/claude-agent-sdk/claude-agent-sdk.ts +++ b/js/src/wrappers/claude-agent-sdk/claude-agent-sdk.ts @@ -1,6 +1,6 @@ -import { claudeAgentSDKChannels } from "../../instrumentation/plugins/claude-agent-sdk-channels"; -import { CLAUDE_AGENT_SDK_SKIP_LOCAL_TOOL_HOOKS_OPTION } from "../../instrumentation/plugins/claude-agent-sdk-instrumentation-constants"; -import { wrapLocalClaudeToolHandler } from "../../instrumentation/plugins/claude-agent-sdk-local-tool-spans"; +import { claudeAgentSDKChannels } from "../../instrumentation/providers/claude-agent-sdk-channels"; +import { CLAUDE_AGENT_SDK_SKIP_LOCAL_TOOL_HOOKS_OPTION } from "../../instrumentation/providers/claude-agent-sdk-instrumentation-constants"; +import { wrapLocalClaudeToolHandler } from "../../instrumentation/providers/claude-agent-sdk-local-tool-spans"; import type { ClaudeAgentSDKModule, ClaudeAgentSDKQueryParams, diff --git a/js/src/wrappers/cloudflare-agent.ts b/js/src/wrappers/cloudflare-agent.ts index c34bb6134..baee1d63d 100644 --- a/js/src/wrappers/cloudflare-agent.ts +++ b/js/src/wrappers/cloudflare-agent.ts @@ -1,5 +1,5 @@ import { debugLogger } from "../debug-logger"; -import { cloudflareAgentsChannels } from "../instrumentation/plugins/cloudflare-agents-channels"; +import { cloudflareAgentsChannels } from "../instrumentation/providers/cloudflare-agents-channels"; import type { CloudflareAgent } from "../vendor-sdk-types/cloudflare-agents"; const WRAPPED_RUN_AGENT_TOOL = Symbol.for( diff --git a/js/src/wrappers/cloudflare-ai-chat.ts b/js/src/wrappers/cloudflare-ai-chat.ts index 87df27775..11f834feb 100644 --- a/js/src/wrappers/cloudflare-ai-chat.ts +++ b/js/src/wrappers/cloudflare-ai-chat.ts @@ -1,5 +1,5 @@ import { debugLogger } from "../debug-logger"; -import { instrumentCloudflareAIChatAgent } from "../instrumentation/plugins/cloudflare-ai-chat-instrumentation"; +import { instrumentCloudflareAIChatAgent } from "../instrumentation/providers/cloudflare-ai-chat-instrumentation"; import type { CloudflareAIChatAgent, CloudflareAIChatAgentConstructor, diff --git a/js/src/wrappers/cloudflare-think.ts b/js/src/wrappers/cloudflare-think.ts index 9c04400ce..2b1e371bd 100644 --- a/js/src/wrappers/cloudflare-think.ts +++ b/js/src/wrappers/cloudflare-think.ts @@ -1,4 +1,4 @@ -import { cloudflareThinkChannels } from "../instrumentation/plugins/cloudflare-think-channels"; +import { cloudflareThinkChannels } from "../instrumentation/providers/cloudflare-think-channels"; import type { CloudflareThinkConstructor, CloudflareThinkInstance, diff --git a/js/src/wrappers/cohere.ts b/js/src/wrappers/cohere.ts index aa6038030..6c55f65dd 100644 --- a/js/src/wrappers/cohere.ts +++ b/js/src/wrappers/cohere.ts @@ -1,4 +1,4 @@ -import { cohereChannels } from "../instrumentation/plugins/cohere-channels"; +import { cohereChannels } from "../instrumentation/providers/cohere-channels"; import type { CohereChatRequest, CohereChatResponse, diff --git a/js/src/wrappers/cursor-sdk.ts b/js/src/wrappers/cursor-sdk.ts index e20f41fba..c20897b82 100644 --- a/js/src/wrappers/cursor-sdk.ts +++ b/js/src/wrappers/cursor-sdk.ts @@ -1,4 +1,4 @@ -import { cursorSDKChannels } from "../instrumentation/plugins/cursor-sdk-channels"; +import { cursorSDKChannels } from "../instrumentation/providers/cursor-sdk-channels"; import type { CursorSDKAgent, CursorSDKAgentClass, diff --git a/js/src/wrappers/genkit.test.ts b/js/src/wrappers/genkit.test.ts index 5363a64eb..3677dc99d 100644 --- a/js/src/wrappers/genkit.test.ts +++ b/js/src/wrappers/genkit.test.ts @@ -1,7 +1,7 @@ import { afterEach, describe, expect, it, vi } from "vitest"; import type { IsoChannelHandlers } from "../isomorph"; import type { ChannelMessage } from "../instrumentation/core/channel-definitions"; -import { genkitChannels } from "../instrumentation/plugins/genkit-channels"; +import { genkitChannels } from "../instrumentation/providers/genkit-channels"; import { configureNode } from "../node/config"; import type { GenkitAction, diff --git a/js/src/wrappers/genkit.ts b/js/src/wrappers/genkit.ts index fc35a8426..a4900ae9f 100644 --- a/js/src/wrappers/genkit.ts +++ b/js/src/wrappers/genkit.ts @@ -1,4 +1,4 @@ -import { genkitChannels } from "../instrumentation/plugins/genkit-channels"; +import { genkitChannels } from "../instrumentation/providers/genkit-channels"; import type { GenkitAction, GenkitEmbedManyParams, diff --git a/js/src/wrappers/github-copilot.ts b/js/src/wrappers/github-copilot.ts index 35e68fa87..20e76dbda 100644 --- a/js/src/wrappers/github-copilot.ts +++ b/js/src/wrappers/github-copilot.ts @@ -1,4 +1,4 @@ -import { gitHubCopilotChannels } from "../instrumentation/plugins/github-copilot-channels"; +import { gitHubCopilotChannels } from "../instrumentation/providers/github-copilot-channels"; import type { GitHubCopilotClient, GitHubCopilotResumeSessionConfig, diff --git a/js/src/wrappers/google-adk.ts b/js/src/wrappers/google-adk.ts index 28af7395e..6e908222e 100644 --- a/js/src/wrappers/google-adk.ts +++ b/js/src/wrappers/google-adk.ts @@ -1,4 +1,4 @@ -import { googleADKChannels } from "../instrumentation/plugins/google-adk-channels"; +import { googleADKChannels } from "../instrumentation/providers/google-adk-channels"; import type { GoogleADKRunner, GoogleADKRunnerConstructor, diff --git a/js/src/wrappers/google-genai.ts b/js/src/wrappers/google-genai.ts index 192f79c1a..7d21e8eea 100644 --- a/js/src/wrappers/google-genai.ts +++ b/js/src/wrappers/google-genai.ts @@ -1,4 +1,4 @@ -import { googleGenAIChannels } from "../instrumentation/plugins/google-genai-channels"; +import { googleGenAIChannels } from "../instrumentation/providers/google-genai-channels"; import { isObject } from "../util"; import type { GoogleGenAIClient, diff --git a/js/src/wrappers/groq.ts b/js/src/wrappers/groq.ts index 68312fa81..c736e00c8 100644 --- a/js/src/wrappers/groq.ts +++ b/js/src/wrappers/groq.ts @@ -1,4 +1,4 @@ -import { groqChannels } from "../instrumentation/plugins/groq-channels"; +import { groqChannels } from "../instrumentation/providers/groq-channels"; import type { GroqChat, GroqChatCompletion, diff --git a/js/src/wrappers/huggingface-transformers.ts b/js/src/wrappers/huggingface-transformers.ts index 3965f1234..52d8e2d56 100644 --- a/js/src/wrappers/huggingface-transformers.ts +++ b/js/src/wrappers/huggingface-transformers.ts @@ -1,7 +1,7 @@ import { huggingFaceTransformersChannels, isSupportedHuggingFaceTransformersTask, -} from "../instrumentation/plugins/huggingface-transformers-channels"; +} from "../instrumentation/providers/huggingface-transformers-channels"; import type { HuggingFaceTransformersModule, HuggingFaceTransformersPipeline, diff --git a/js/src/wrappers/huggingface.ts b/js/src/wrappers/huggingface.ts index 6ff98a259..13a58f515 100644 --- a/js/src/wrappers/huggingface.ts +++ b/js/src/wrappers/huggingface.ts @@ -1,4 +1,4 @@ -import { huggingFaceChannels } from "../instrumentation/plugins/huggingface-channels"; +import { huggingFaceChannels } from "../instrumentation/providers/huggingface-channels"; import { isObject } from "../../util"; import type { HuggingFaceChatCompletion, diff --git a/js/src/wrappers/langsmith.ts b/js/src/wrappers/langsmith.ts index a8486c573..983f42dd8 100644 --- a/js/src/wrappers/langsmith.ts +++ b/js/src/wrappers/langsmith.ts @@ -1,5 +1,5 @@ import { debugLogger } from "../debug-logger"; -import { langSmithChannels } from "../instrumentation/plugins/langsmith-channels"; +import { langSmithChannels } from "../instrumentation/providers/langsmith-channels"; import type { LangSmithClient, LangSmithClientConstructor, diff --git a/js/src/wrappers/mistral.ts b/js/src/wrappers/mistral.ts index c449997f7..4771dda38 100644 --- a/js/src/wrappers/mistral.ts +++ b/js/src/wrappers/mistral.ts @@ -1,4 +1,4 @@ -import { mistralChannels } from "../instrumentation/plugins/mistral-channels"; +import { mistralChannels } from "../instrumentation/providers/mistral-channels"; import type { MistralAgents, MistralAgentsCompletionResponse, diff --git a/js/src/wrappers/oai.ts b/js/src/wrappers/oai.ts index b2fa0850e..743460b7e 100644 --- a/js/src/wrappers/oai.ts +++ b/js/src/wrappers/oai.ts @@ -10,7 +10,7 @@ import type { ArgsOf, ResultOf, } from "../instrumentation/core/channel-definitions"; -import { openAIChannels } from "../instrumentation/plugins/openai-channels"; +import { openAIChannels } from "../instrumentation/providers/openai-channels"; import type { OpenAIChatCompletion, OpenAIChatCreateParams, diff --git a/js/src/wrappers/oai_responses.ts b/js/src/wrappers/oai_responses.ts index 8e936ebbc..432b1281a 100644 --- a/js/src/wrappers/oai_responses.ts +++ b/js/src/wrappers/oai_responses.ts @@ -3,7 +3,7 @@ import type { ResultOf, } from "../instrumentation/core/channel-definitions"; import type { ChannelSpanInfo } from "../instrumentation/core/types"; -import { openAIChannels } from "../instrumentation/plugins/openai-channels"; +import { openAIChannels } from "../instrumentation/providers/openai-channels"; import { parseMetricsFromUsage } from "../openai-utils"; import { APIPromise, diff --git a/js/src/wrappers/ollama.test.ts b/js/src/wrappers/ollama.test.ts index 9a8ebff46..b07dac5f4 100644 --- a/js/src/wrappers/ollama.test.ts +++ b/js/src/wrappers/ollama.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it, vi } from "vitest"; import { debugLogger } from "../debug-logger"; -import { ollamaChannels } from "../instrumentation/plugins/ollama-channels"; +import { ollamaChannels } from "../instrumentation/providers/ollama-channels"; import type { OllamaClient } from "../vendor-sdk-types/ollama"; import { wrapOllama } from "./ollama"; diff --git a/js/src/wrappers/ollama.ts b/js/src/wrappers/ollama.ts index 3f90067a9..bcc48a937 100644 --- a/js/src/wrappers/ollama.ts +++ b/js/src/wrappers/ollama.ts @@ -1,5 +1,5 @@ import { debugLogger } from "../debug-logger"; -import { ollamaChannels } from "../instrumentation/plugins/ollama-channels"; +import { ollamaChannels } from "../instrumentation/providers/ollama-channels"; import { isObject } from "../../util"; import type { OllamaChatRequest, diff --git a/js/src/wrappers/openai-codex.ts b/js/src/wrappers/openai-codex.ts index 8b5a435b3..c4225e9c0 100644 --- a/js/src/wrappers/openai-codex.ts +++ b/js/src/wrappers/openai-codex.ts @@ -1,4 +1,4 @@ -import { openAICodexChannels } from "../instrumentation/plugins/openai-codex-channels"; +import { openAICodexChannels } from "../instrumentation/providers/openai-codex-channels"; import type { OpenAICodexClass, OpenAICodexClient, diff --git a/js/src/wrappers/openai-promise-utils.ts b/js/src/wrappers/openai-promise-utils.ts index afa23586c..b41e57f4b 100644 --- a/js/src/wrappers/openai-promise-utils.ts +++ b/js/src/wrappers/openai-promise-utils.ts @@ -6,7 +6,7 @@ import type { OpenAIAsyncChannel, OpenAIChannel, OpenAIStartContext, -} from "../instrumentation/plugins/openai-channels"; +} from "../instrumentation/providers/openai-channels"; export type EnhancedResponse = { response: Response; diff --git a/js/src/wrappers/openrouter-agent.test.ts b/js/src/wrappers/openrouter-agent.test.ts index 02f01e040..c1c94e12f 100644 --- a/js/src/wrappers/openrouter-agent.test.ts +++ b/js/src/wrappers/openrouter-agent.test.ts @@ -1,6 +1,6 @@ import { afterEach, describe, expect, it, vi } from "vitest"; import { wrapOpenRouterAgent } from "./openrouter-agent"; -import { openRouterAgentChannels } from "../instrumentation/plugins/openrouter-agent-channels"; +import { openRouterAgentChannels } from "../instrumentation/providers/openrouter-agent-channels"; describe("wrapOpenRouterAgent", () => { afterEach(() => { diff --git a/js/src/wrappers/openrouter-agent.ts b/js/src/wrappers/openrouter-agent.ts index 07328249b..336b2d84f 100644 --- a/js/src/wrappers/openrouter-agent.ts +++ b/js/src/wrappers/openrouter-agent.ts @@ -1,4 +1,4 @@ -import { openRouterAgentChannels } from "../instrumentation/plugins/openrouter-agent-channels"; +import { openRouterAgentChannels } from "../instrumentation/providers/openrouter-agent-channels"; import type { OpenRouterAgentClient, OpenRouterAgentCallModelRequest, diff --git a/js/src/wrappers/openrouter.ts b/js/src/wrappers/openrouter.ts index bfd39d2c1..ab7231b0c 100644 --- a/js/src/wrappers/openrouter.ts +++ b/js/src/wrappers/openrouter.ts @@ -1,4 +1,4 @@ -import { openRouterChannels } from "../instrumentation/plugins/openrouter-channels"; +import { openRouterChannels } from "../instrumentation/providers/openrouter-channels"; import type { OpenRouterBeta, OpenRouterCallModelRequest, diff --git a/js/src/wrappers/pi-coding-agent.ts b/js/src/wrappers/pi-coding-agent.ts index 60565c596..f01f6a75a 100644 --- a/js/src/wrappers/pi-coding-agent.ts +++ b/js/src/wrappers/pi-coding-agent.ts @@ -1,4 +1,4 @@ -import { piCodingAgentChannels } from "../instrumentation/plugins/pi-coding-agent-channels"; +import { piCodingAgentChannels } from "../instrumentation/providers/pi-coding-agent-channels"; import type { PiAgentSession, PiAgentSessionClass, diff --git a/js/src/wrappers/strands-agent-sdk.ts b/js/src/wrappers/strands-agent-sdk.ts index 6c6173034..3b975c75c 100644 --- a/js/src/wrappers/strands-agent-sdk.ts +++ b/js/src/wrappers/strands-agent-sdk.ts @@ -1,4 +1,4 @@ -import { strandsAgentSDKChannels } from "../instrumentation/plugins/strands-agent-sdk-channels"; +import { strandsAgentSDKChannels } from "../instrumentation/providers/strands-agent-sdk-channels"; import type { StrandsAgent, StrandsAgentConstructor, diff --git a/js/src/wrappers/voyageai.ts b/js/src/wrappers/voyageai.ts index 6abc4bb1a..8285282f5 100644 --- a/js/src/wrappers/voyageai.ts +++ b/js/src/wrappers/voyageai.ts @@ -1,4 +1,4 @@ -import { voyageAIChannels } from "../instrumentation/plugins/voyageai-channels"; +import { voyageAIChannels } from "../instrumentation/providers/voyageai-channels"; import type { VoyageAIClient, VoyageAIContextualizedEmbedRequest, From 34bff9d12fda835815f06c76c0d9cf998c7384b1 Mon Sep 17 00:00:00 2001 From: Luca Forstner Date: Mon, 31 Aug 2026 20:05:47 +0200 Subject: [PATCH 7/8] ref!: Remove `flushAsync` option (#2409) Ref [braintrustdata/braintrust-sdk-javascript#1841](https://github.com/braintrustdata/braintrust-sdk-javascript/issues/1841) `flushAsync` grinded my gears a bit a) as a user b) as a maintainer. Why?: - As a user, it added a quite intransparent API that is hard to wrap your head around. - As a user, it added a generic type to otherwise very non-generic APIs that made the LSP types incredibly hard to navigate - As a maintainer, it required a generic type to be passed around to literally everywhere - As a maintainer, it required us to have a weird `flushAsync` option to every tracing API In reality, what you can do instead, and which is very simple to understand, is simply do `await flush()` or `await logger.flush()` right after a span is ended. Not hard, very few if no drawbacks. --- .changeset/remove-flush-async.md | 5 + e2e/helpers/prod-forwarding.ts | 1 - .../src/BraintrustCallbackHandler.ts | 15 +- integrations/openai-agents-js/src/index.ts | 2 +- .../src/openai-agents-integration.test.ts | 2 +- integrations/openai-agents-js/src/types.ts | 2 +- js/src/exports.ts | 3 - .../providers/eve-instrumentation.ts | 4 +- .../openai-agents-trace-processor.ts | 4 +- js/src/logger.test.ts | 70 ++- js/src/logger.ts | 410 ++++++------------ js/src/vendor-sdk-types/langchain.ts | 4 +- js/src/wrappers/langchain/callback-handler.ts | 10 +- js/src/wrappers/test_oai_attachments.test.ts | 1 - .../api-compatibility.test.ts | 28 +- 15 files changed, 230 insertions(+), 331 deletions(-) create mode 100644 .changeset/remove-flush-async.md diff --git a/.changeset/remove-flush-async.md b/.changeset/remove-flush-async.md new file mode 100644 index 000000000..18903d6a9 --- /dev/null +++ b/.changeset/remove-flush-async.md @@ -0,0 +1,5 @@ +--- +"braintrust": major +--- + +ref!: Remove `flushAsync` option diff --git a/e2e/helpers/prod-forwarding.ts b/e2e/helpers/prod-forwarding.ts index 522aedbe6..37f08d14f 100644 --- a/e2e/helpers/prod-forwarding.ts +++ b/e2e/helpers/prod-forwarding.ts @@ -31,7 +31,6 @@ export async function initializeProdForwarding(): Promise { const logger = initLogger({ apiKey, appUrl: process.env.BRAINTRUST_APP_URL, - asyncFlush: false, forceLogin: true, projectName, }); diff --git a/integrations/langchain-js/src/BraintrustCallbackHandler.ts b/integrations/langchain-js/src/BraintrustCallbackHandler.ts index d53b5ef69..56855154d 100644 --- a/integrations/langchain-js/src/BraintrustCallbackHandler.ts +++ b/integrations/langchain-js/src/BraintrustCallbackHandler.ts @@ -11,23 +11,18 @@ import { type LangChainCallbackHandlerOptions, } from "braintrust"; -type BraintrustCallbackHandlerOptions = - LangChainCallbackHandlerOptions; +type BraintrustCallbackHandlerOptions = LangChainCallbackHandlerOptions; /** * A Braintrust tracer for LangChain.js that logs LLM calls, chains, and tools. */ -export class BraintrustCallbackHandler< - IsAsyncFlush extends boolean, -> extends BaseCallbackHandler { +export class BraintrustCallbackHandler extends BaseCallbackHandler { name = "BraintrustCallbackHandler"; - private inner: BraintrustLangChainCallbackHandler; + private inner: BraintrustLangChainCallbackHandler; - constructor( - options?: Partial>, - ) { + constructor(options?: Partial) { super(); - this.inner = new BraintrustLangChainCallbackHandler(options); + this.inner = new BraintrustLangChainCallbackHandler(options); } handleLLMStart( diff --git a/integrations/openai-agents-js/src/index.ts b/integrations/openai-agents-js/src/index.ts index 394d61727..beafc3fc0 100644 --- a/integrations/openai-agents-js/src/index.ts +++ b/integrations/openai-agents-js/src/index.ts @@ -118,7 +118,7 @@ function getTimeElapsed(end?: string, start?: string): number | undefined { export class OpenAIAgentsTraceProcessor { private static readonly DEFAULT_MAX_TRACES = 10000; - private logger?: Logger; + private logger?: Logger; private maxTraces: number; private traceSpans = new Map< string, diff --git a/integrations/openai-agents-js/src/openai-agents-integration.test.ts b/integrations/openai-agents-js/src/openai-agents-integration.test.ts index 317d04b7e..71a6e5fd5 100644 --- a/integrations/openai-agents-js/src/openai-agents-integration.test.ts +++ b/integrations/openai-agents-js/src/openai-agents-integration.test.ts @@ -55,7 +55,7 @@ describe( TEST_SUITE_OPTIONS, () => { let backgroundLogger: TestBackgroundLogger; - let _logger: Logger; + let _logger: Logger; let Agent: any; let run: any; let tool: any; diff --git a/integrations/openai-agents-js/src/types.ts b/integrations/openai-agents-js/src/types.ts index 6834f5111..15aea9a6f 100644 --- a/integrations/openai-agents-js/src/types.ts +++ b/integrations/openai-agents-js/src/types.ts @@ -50,7 +50,7 @@ export type TraceMetadata = { }; export interface OpenAIAgentsTraceProcessorOptions { - logger?: Logger; + logger?: Logger; maxTraces?: number; } diff --git a/js/src/exports.ts b/js/src/exports.ts index 487ec7430..ffce08390 100644 --- a/js/src/exports.ts +++ b/js/src/exports.ts @@ -31,13 +31,11 @@ export type { InitLoggerOptions, InitOptions, LoadPromptOptions, - LogOptions, LoginOptions, Logs3OverflowInputRow, Logs3OverflowUpload, MetricSummary, ObjectMetadata, - PromiseUnless, PropagationContext, PromptRowWithId, ScoreSummary, @@ -125,7 +123,6 @@ export { withCurrent, withDataset, withExperiment, - withLogger, withParent, wrapTraced, registerOtelFlush, diff --git a/js/src/instrumentation/providers/eve-instrumentation.ts b/js/src/instrumentation/providers/eve-instrumentation.ts index 8729c6a48..610083878 100644 --- a/js/src/instrumentation/providers/eve-instrumentation.ts +++ b/js/src/instrumentation/providers/eve-instrumentation.ts @@ -585,7 +585,7 @@ class EveBridge { turn: TurnState, event: { input?: unknown; output?: unknown }, ): void { - currentLogger()?.updateSpan({ + currentLogger()?.updateSpan({ id: turn.rowId, root_span_id: turn.rootSpanId, span_id: turn.spanId, @@ -637,7 +637,7 @@ class EveBridge { if (Object.keys(totals).length === 0) { return; } - currentLogger()?.updateSpan({ + currentLogger()?.updateSpan({ id: turn.rowId, metrics: totals, root_span_id: turn.rootSpanId, diff --git a/js/src/instrumentation/providers/openai-agents-trace-processor.ts b/js/src/instrumentation/providers/openai-agents-trace-processor.ts index e3116ed58..a335f2efb 100644 --- a/js/src/instrumentation/providers/openai-agents-trace-processor.ts +++ b/js/src/instrumentation/providers/openai-agents-trace-processor.ts @@ -44,7 +44,7 @@ type TraceMetadata = { }; export interface OpenAIAgentsTraceProcessorOptions { - logger?: Logger; + logger?: Logger; maxTraces?: number; } @@ -180,7 +180,7 @@ function parseUsageMetrics(usage: unknown): Record { export class OpenAIAgentsTraceProcessor { private static readonly DEFAULT_MAX_TRACES = 10000; - private logger?: Logger; + private logger?: Logger; private maxTraces: number; private traceSpans = new Map< string, diff --git a/js/src/logger.test.ts b/js/src/logger.test.ts index 86a3a1a5c..1dffdbb49 100644 --- a/js/src/logger.test.ts +++ b/js/src/logger.test.ts @@ -1,6 +1,14 @@ /* eslint-disable @typescript-eslint/consistent-type-assertions */ -import { vi, expect, test, describe, beforeEach, afterEach } from "vitest"; +import { + vi, + expect, + expectTypeOf, + test, + describe, + beforeEach, + afterEach, +} from "vitest"; import { _exportsForTestingOnly, init, @@ -11,6 +19,7 @@ import { loadPrompt, loadParameters, wrapTraced, + traced, currentSpan, withParent, startSpan, @@ -19,6 +28,8 @@ import { deepCopyEvent, ReadonlyExperiment, renderMessageImpl, + flush, + type InitLoggerOptions, } from "./logger"; import { configureNode } from "./node/config"; @@ -31,6 +42,12 @@ import { SpanCache } from "./span-cache"; configureNode(); +type InitLoggerHasAsyncFlush = "asyncFlush" extends keyof InitLoggerOptions + ? true + : false; + +expectTypeOf().toEqualTypeOf(); + test("renderMessage with file content parts", () => { const message = { role: "user" as const, @@ -426,6 +443,57 @@ test("verify MemoryBackgroundLogger intercepts logs", async () => { _exportsForTestingOnly.clearTestBackgroundLogger(); // can go back to normal }); +test("logger and tracing APIs preserve return shapes without implicit flushing", async () => { + await _exportsForTestingOnly.simulateLoginForTests(); + const backgroundLogger = _exportsForTestingOnly.useTestBackgroundLogger(); + const flushSpy = vi.spyOn(backgroundLogger, "flush"); + + try { + const logger = initLogger({ + projectName: "test", + projectId: "test-project-id", + }); + + const id = logger.log({ input: "input" }); + expectTypeOf(id).toEqualTypeOf(); + + const loggerSyncResult = logger.traced(() => 42 as const); + expectTypeOf(loggerSyncResult).toEqualTypeOf<42>(); + expect(loggerSyncResult).toBe(42); + + const loggerAsyncResult = logger.traced(async () => 43); + expectTypeOf(loggerAsyncResult).toEqualTypeOf>(); + await expect(loggerAsyncResult).resolves.toBe(43); + + const tracedSyncResult = traced(() => "sync" as const); + expectTypeOf(tracedSyncResult).toEqualTypeOf<"sync">(); + expect(tracedSyncResult).toBe("sync"); + + const syncFunction = (value: number) => value + 1; + const wrappedSyncFunction = wrapTraced(syncFunction); + expectTypeOf(wrappedSyncFunction).toEqualTypeOf(); + expect(wrappedSyncFunction(1)).toBe(2); + + const asyncFunction = async (value: number) => value + 1; + const wrappedAsyncFunction = wrapTraced(asyncFunction); + expectTypeOf(wrappedAsyncFunction).toEqualTypeOf(); + await expect(wrappedAsyncFunction(2)).resolves.toBe(3); + + expect(flushSpy).not.toHaveBeenCalled(); + + await logger.flush(); + const span = logger.startSpan({ name: "explicit-flush" }); + span.end(); + await span.flush(); + await flush(); + + expect(flushSpy).toHaveBeenCalledTimes(3); + } finally { + _exportsForTestingOnly.clearTestBackgroundLogger(); + _exportsForTestingOnly.simulateLogoutForTests(); + } +}); + test("init validation", () => { expect(() => init({})).toThrow( "Must specify at least one of project or projectId", diff --git a/js/src/logger.ts b/js/src/logger.ts index fcf88d9a4..793cb96fc 100644 --- a/js/src/logger.ts +++ b/js/src/logger.ts @@ -764,9 +764,7 @@ function normalizeProxyConnUrl(proxyUrl: string): string { export class BraintrustState { public id: string; public currentExperiment: Experiment | undefined; - // Note: the value of IsAsyncFlush doesn't really matter here, since we - // (safely) dynamically cast it whenever retrieving the logger. - public currentLogger: Logger | undefined; + public currentLogger: Logger | undefined; public currentParent: IsoAsyncLocalStorage; public currentSpan: IsoAsyncLocalStorage; // Any time we re-log in, we directly update the apiConn inside the logger. @@ -1531,20 +1529,15 @@ interface OrgProjectMetadata { project: ObjectMetadata; } -export interface LinkArgs { +interface LinkArgs { org_name?: string; app_url?: string; - project_name?: string; - project_id?: string; -} - -export interface LogOptions { - asyncFlush?: IsAsyncFlush; - computeMetadataArgs?: Record; - linkArgs?: LinkArgs; } -export type PromiseUnless = B extends true ? R : Promise>; +type ProjectMetadataArgs = { + project_name?: string; + project_id?: string; +}; export interface AttachmentParams { data: string | Blob | ArrayBuffer; @@ -2546,12 +2539,11 @@ function startSpanParentArgs(args: { }; } -export class Logger implements Exportable { +export class Logger implements Exportable { private state: BraintrustState; private lazyMetadata: LazyValue; - private _asyncFlush: IsAsyncFlush | undefined; - private computeMetadataArgs: Record | undefined; - private _linkArgs: LinkArgs | undefined; + private computeMetadataArgs: ProjectMetadataArgs | undefined; + private linkArgs: LinkArgs | undefined; private lastStartTime: number; private lazyId: LazyValue; private calledStartSpan: boolean; @@ -2562,14 +2554,16 @@ export class Logger implements Exportable { constructor( state: BraintrustState, lazyMetadata: LazyValue, - logOptions: LogOptions = {}, + options: { + computeMetadataArgs?: ProjectMetadataArgs; + linkArgs?: LinkArgs; + } = {}, ) { this.lazyMetadata = lazyMetadata; - this._asyncFlush = logOptions.asyncFlush; - this.computeMetadataArgs = logOptions.computeMetadataArgs; - this._linkArgs = logOptions.linkArgs; + this.computeMetadataArgs = options.computeMetadataArgs; + this.linkArgs = options.linkArgs; this.lastStartTime = getCurrentUnixTimestamp(); - this.lazyId = new LazyValue(async () => await this.id); + this.lazyId = new LazyValue(() => this.id); this.calledStartSpan = false; this.state = state; } @@ -2599,7 +2593,7 @@ export class Logger implements Exportable { } /** - * Log a single event. The event will be batched and uploaded behind the scenes if `logOptions.asyncFlush` is true. + * Log a single event. The event will be batched and uploaded behind the scenes. Call and await {@link Logger.flush} to ensure the event has been uploaded. * * @param event The event to log. * @param event.input: (Optional) the arguments that uniquely define a user input (an arbitrary, JSON serializable object). @@ -2617,7 +2611,7 @@ export class Logger implements Exportable { public log( event: Readonly, options?: { allowConcurrentWithSpans?: boolean }, - ): PromiseUnless { + ): string { if (this.calledStartSpan && !options?.allowConcurrentWithSpans) { throw new Error( "Cannot run toplevel `log` method while using spans. To log to the span, call `logger.traced` and then log with `span.log`", @@ -2626,16 +2620,7 @@ export class Logger implements Exportable { const span = this.startSpanImpl({ startTime: this.lastStartTime, event }); this.lastStartTime = span.end(); - const ret = span.id; - type Ret = PromiseUnless; - if (this.asyncFlush === true) { - return ret as Ret; - } else { - return (async () => { - await this.flush(); - return ret; - })() as Ret; - } + return span.id; } /** @@ -2646,11 +2631,11 @@ export class Logger implements Exportable { public traced( callback: (span: Span) => R, args?: StartSpanArgs & SetCurrentArg, - ): PromiseUnless { + ): R { const { setCurrent, ...argsRest } = args ?? {}; const span = this.startSpan(argsRest); - const ret = runCatchFinally( + return runCatchFinally( () => { if (setCurrent ?? true) { return withCurrent(span, callback); @@ -2658,23 +2643,12 @@ export class Logger implements Exportable { return callback(span); } }, - (e) => { - logError(span, e); - throw e; + (error) => { + logError(span, error); + throw error; }, () => span.end(), ); - type Ret = PromiseUnless; - - if (this.asyncFlush) { - return ret as Ret; - } else { - return (async () => { - const awaitedRet = await ret; - await this.flush(); - return awaitedRet; - })() as Ret; - } } /** @@ -2773,17 +2747,13 @@ export class Logger implements Exportable { return await this.state.bgLogger().flush(); } - get asyncFlush(): IsAsyncFlush | undefined { - return this._asyncFlush; - } - /** * Return the base URL for links (e.g. https://braintrust.dev/app/my-org-name) * if we have the info, otherwise return null. * Resolution order: state -> linkArgs -> env var */ public _getLinkBaseUrl(): string | null { - return _getLinkBaseUrl(this.state, this._linkArgs); + return _getLinkBaseUrl(this.state, this.linkArgs); } /** @@ -2805,19 +2775,6 @@ export class Logger implements Exportable { } } -function castLogger( - logger: Logger | undefined, - asyncFlush?: ToB, -): Logger | undefined { - if (logger === undefined) return undefined; - if (asyncFlush !== undefined && !!asyncFlush !== !!logger.asyncFlush) { - throw new Error( - `Asserted asyncFlush setting ${asyncFlush} does not match stored logger's setting ${logger.asyncFlush}`, - ); - } - return logger as unknown as Logger; -} - export type Logs3OverflowUpload = { method: "PUT" | "POST"; signedUrl: string; @@ -4210,22 +4167,6 @@ export function withExperiment( return callback(experiment); } -/** - * @deprecated Use {@link initLogger} instead. - */ -export function withLogger( - callback: (logger: Logger) => R, - options: Readonly & SetCurrentArg> = {}, -): R { - debugLogger - .forState(options.state) - .warn( - "withLogger is deprecated and will be removed in a future version of braintrust. Simply create the logger with `initLogger`.", - ); - const logger = initLogger(options); - return callback(logger); -} - type UseOutputOption = { useOutput?: IsLegacyDataset; }; @@ -4675,13 +4616,7 @@ export function withDataset< // from arguments serialized elsewhere. async function computeLoggerMetadata( state: BraintrustState, - { - project_name, - project_id, - }: { - project_name?: string; - project_id?: string; - }, + { project_name, project_id }: ProjectMetadataArgs, ) { await state.login({}); const org_id = state.orgId!; @@ -4718,18 +4653,13 @@ async function computeLoggerMetadata( } } -type AsyncFlushArg = { - asyncFlush?: IsAsyncFlush; -}; - -export type InitLoggerOptions = FullLoginOptions & { +export type InitLoggerOptions = FullLoginOptions & { projectName?: string; projectId?: string; environment?: SpanOriginEnvironment; setCurrent?: boolean; state?: BraintrustState; - orgProjectMetadata?: OrgProjectMetadata; -} & AsyncFlushArg; +}; /** * Create a new logger in a specified project. If the project does not exist, it will be created. @@ -4737,7 +4667,6 @@ export type InitLoggerOptions = FullLoginOptions & { * @param options Additional options for configuring init(). * @param options.projectName The name of the project to log into. If unspecified, will default to the Global project. * @param options.projectId The id of the project to log into. This takes precedence over projectName if specified. - * @param options.asyncFlush If true, will log asynchronously in the background. Otherwise, will log synchronously. (true by default) * @param options.appUrl The URL of the Braintrust App. Defaults to https://www.braintrust.dev. * @param options.apiKey The API key to use. If the parameter is not specified, will try to use the `BRAINTRUST_API_KEY` environment variable. In Node.js, * if that is unset, will try the nearest `.env.braintrust` file in the current working directory or parent directories. If no API key is specified, will prompt the user to login. @@ -4747,13 +4676,10 @@ export type InitLoggerOptions = FullLoginOptions & { * @param setCurrent If true (the default), set the global current-experiment to the newly-created one. * @returns The newly created Logger. */ -export function initLogger( - options: Readonly> = {}, -) { +export function initLogger(options: Readonly = {}): Logger { const { projectName, projectId, - asyncFlush: asyncFlushArg, appUrl, apiKey, orgName, @@ -4764,9 +4690,6 @@ export function initLogger( state: stateArg, } = options || {}; - const asyncFlush = - asyncFlushArg === undefined ? (true as IsAsyncFlush) : asyncFlushArg; - const computeMetadataArgs = { project_name: projectName, project_id: projectId, @@ -4775,8 +4698,6 @@ export function initLogger( const linkArgs = { org_name: orgName, app_url: appUrl, - project_name: projectName, - project_id: projectId, }; const state = stateArg ?? _globalState; @@ -4801,13 +4722,12 @@ export function initLogger( }, ); - const ret = new Logger(state, lazyMetadata, { - asyncFlush, + const ret = new Logger(state, lazyMetadata, { computeMetadataArgs, linkArgs, }); if (options.setCurrent ?? true) { - state.currentLogger = ret as Logger; + state.currentLogger = ret; } return ret; } @@ -5484,11 +5404,9 @@ export function currentExperiment( /** * Returns the currently-active logger (set by {@link initLogger}). Returns undefined if no current logger has been set. */ -export function currentLogger( - options?: AsyncFlushArg & OptionalStateArg, -): Logger | undefined { +export function currentLogger(options?: OptionalStateArg): Logger | undefined { const state = options?.state ?? _globalState; - return castLogger(state.currentLogger, options?.asyncFlush); + return state.currentLogger; } /** @@ -5511,16 +5429,15 @@ export function currentSpan(options?: OptionalStateArg): Span { * disagreeing if state changed between calls). The state is only meaningful when * a parent slug was resolved; otherwise it is undefined. */ -function getSpanParentObjectAndPropagatedState( - options?: AsyncFlushArg & - OptionalStateArg & { parent?: string | PropagationContext }, +function getSpanParentObjectAndPropagatedState( + options?: OptionalStateArg & { parent?: string | PropagationContext }, ): { parentObject: | SpanComponentsV3 | SpanComponentsV4 | Span | Experiment - | Logger; + | Logger; propagatedState: PropagatedState | undefined; } { const state = options?.state ?? _globalState; @@ -5539,11 +5456,11 @@ function getSpanParentObjectAndPropagatedState( }; } - const experiment = currentExperiment(); + const experiment = state.currentExperiment; if (experiment) { return { parentObject: experiment, propagatedState: undefined }; } - const logger = currentLogger(options); + const logger = state.currentLogger; if (logger) { return { parentObject: logger, propagatedState: undefined }; } @@ -5557,15 +5474,9 @@ function getSpanParentObjectAndPropagatedState( * `parent` may be an exported slug string or an opaque W3C trace-context (from * {@link extractTraceContextFromHeaders}). */ -export function getSpanParentObject( - options?: AsyncFlushArg & - OptionalStateArg & { parent?: string | PropagationContext }, -): - | SpanComponentsV3 - | SpanComponentsV4 - | Span - | Experiment - | Logger { +export function getSpanParentObject( + options?: OptionalStateArg & { parent?: string | PropagationContext }, +): SpanComponentsV3 | SpanComponentsV4 | Span | Experiment | Logger { return getSpanParentObjectAndPropagatedState(options).parentObject; } @@ -6063,45 +5974,33 @@ export function logError(span: Span, error: unknown) { * * See {@link Span.traced} for full details. */ -export function traced( +export function traced( callback: (span: Span) => R, - args?: StartSpanArgs & - SetCurrentArg & - AsyncFlushArg & - OptionalStateArg, -): PromiseUnless { - const { span, isSyncFlushLogger } = startSpanAndIsLogger(args); - - const ret = runCatchFinally( + args?: StartSpanArgs & SetCurrentArg & OptionalStateArg, +): R { + const { setCurrent, ...spanArgs } = args ?? {}; + const span = startSpanImpl(spanArgs); + + return runCatchFinally( () => { - if (args?.setCurrent ?? true) { + if (setCurrent ?? true) { return withCurrent(span, callback); } else { return callback(span); } }, - (e) => { - logError(span, e); - throw e; + (error) => { + logError(span, error); + throw error; }, () => span.end(), ); - - type Ret = PromiseUnless; - - if (args?.asyncFlush === undefined || args?.asyncFlush) { - return ret as Ret; - } else { - return (async () => { - const awaitedRet = await ret; - if (isSyncFlushLogger) { - await span.flush(); - } - return awaitedRet; - })() as Ret; - } } +type WrapTracedArgs = { + noTraceIO?: boolean; +}; + /** * Check if a function is a sync generator function. * @@ -6113,7 +6012,7 @@ export function traced( * @param fn The function to check. * @returns True if the function is a sync generator function. */ -function isGeneratorFunction(fn: any): boolean { +function isGeneratorFunction(fn: unknown): boolean { return Object.prototype.toString.call(fn) === "[object GeneratorFunction]"; } @@ -6124,7 +6023,7 @@ function isGeneratorFunction(fn: any): boolean { * @param fn The function to check. * @returns True if the function is an async generator function. */ -function isAsyncGeneratorFunction(fn: any): boolean { +function isAsyncGeneratorFunction(fn: unknown): boolean { return ( Object.prototype.toString.call(fn) === "[object AsyncGeneratorFunction]" ); @@ -6135,10 +6034,13 @@ function isAsyncGeneratorFunction(fn: any): boolean { */ function wrapTracedSyncGenerator any>( fn: F, - spanArgs: any, + spanArgs: StartSpanArgs & SetCurrentArg, noTraceIO: boolean, ): F { - const wrapper = function* (this: any, ...fnArgs: Parameters) { + const wrapper = function* ( + this: ThisParameterType, + ...fnArgs: Parameters + ) { const span = startSpan(spanArgs); try { if (!noTraceIO) { @@ -6149,7 +6051,7 @@ function wrapTracedSyncGenerator any>( const maxItems = envValue !== undefined ? Number(envValue) : 1000; if (!noTraceIO && maxItems !== 0) { - let collected: any[] = []; + let collected: unknown[] = []; let truncated = false; const gen = generatorWithCurrent(span, fn.apply(this, fnArgs)); @@ -6202,10 +6104,13 @@ function wrapTracedSyncGenerator any>( */ function wrapTracedAsyncGenerator any>( fn: F, - spanArgs: any, + spanArgs: StartSpanArgs & SetCurrentArg, noTraceIO: boolean, ): F { - const wrapper = async function* (this: any, ...fnArgs: Parameters) { + const wrapper = async function* ( + this: ThisParameterType, + ...fnArgs: Parameters + ) { const span = startSpan(spanArgs); try { if (!noTraceIO) { @@ -6216,7 +6121,7 @@ function wrapTracedAsyncGenerator any>( const maxItems = envValue !== undefined ? Number(envValue) : 1000; if (!noTraceIO && maxItems !== 0) { - let collected: any[] = []; + let collected: unknown[] = []; let truncated = false; const gen = asyncGeneratorWithCurrent(span, fn.apply(this, fnArgs)); @@ -6264,10 +6169,6 @@ function wrapTracedAsyncGenerator any>( return wrapper as F; } -type WrapTracedArgs = { - noTraceIO?: boolean; -}; - /** * Wrap a function with `traced`, using the arguments as `input` and return value as `output`. * Any functions wrapped this way will automatically be traced, similar to the `@traced` decorator @@ -6293,84 +6194,52 @@ type WrapTracedArgs = { * @param args Span-level arguments (e.g. a custom name or type) to pass to `traced`. * @returns The wrapped function. */ -export function wrapTraced< - F extends (...args: any[]) => any, - IsAsyncFlush extends boolean = true, ->( +export function wrapTraced any>( fn: F, - args?: StartSpanArgs & - SetCurrentArg & - AsyncFlushArg & - WrapTracedArgs, -): IsAsyncFlush extends false - ? (...args: Parameters) => Promise>> - : F { - const spanArgs: typeof args = { + args?: StartSpanArgs & SetCurrentArg & WrapTracedArgs, +): F { + const { noTraceIO, ...argsRest } = args ?? {}; + const spanArgs: StartSpanArgs & SetCurrentArg = { name: fn.name, type: "function", - ...args, + ...argsRest, }; - const hasExplicitInput = - args && - args.event && - "input" in args.event && - args.event.input !== undefined; - const hasExplicitOutput = - args && args.event && args.event.output !== undefined; - - const noTraceIO = args?.noTraceIO || hasExplicitInput || hasExplicitOutput; + const hasExplicitInput = spanArgs.event?.input !== undefined; + const hasExplicitOutput = spanArgs.event?.output !== undefined; + + const disableGeneratorTraceIO = + !!noTraceIO || hasExplicitInput || hasExplicitOutput; // Check if the function is a generator if (isGeneratorFunction(fn)) { - return wrapTracedSyncGenerator(fn, spanArgs, !!noTraceIO); + return wrapTracedSyncGenerator(fn, spanArgs, disableGeneratorTraceIO); } if (isAsyncGeneratorFunction(fn)) { - return wrapTracedAsyncGenerator(fn, spanArgs, !!noTraceIO); + return wrapTracedAsyncGenerator(fn, spanArgs, disableGeneratorTraceIO); } - if (args?.asyncFlush) { - return ((...fnArgs: Parameters) => - traced((span) => { - if (!hasExplicitInput) { - span.log({ input: fnArgs }); - } - - const output = fn(...fnArgs); - - if (!hasExplicitOutput) { - if (output instanceof Promise) { - return (async () => { - const result = await output; - span.log({ output: result }); - return result; - })(); - } else { - span.log({ output: output }); - } - } - - return output; - }, spanArgs)) as IsAsyncFlush extends false ? never : F; - } else { - return ((...fnArgs: Parameters) => - traced(async (span) => { - if (!hasExplicitInput) { - span.log({ input: fnArgs }); - } - - const outputResult = fn(...fnArgs); + return ((...fnArgs: Parameters) => + traced((span) => { + if (!noTraceIO && !hasExplicitInput) { + span.log({ input: fnArgs }); + } - const output = await outputResult; + const output = fn(...fnArgs); - if (!hasExplicitOutput) { + if (!noTraceIO && !hasExplicitOutput) { + if (output instanceof Promise) { + return (async () => { + const result = await output; + span.log({ output: result }); + return result; + })(); + } else { span.log({ output }); } + } - return output; - }, spanArgs)) as IsAsyncFlush extends false - ? (...args: Parameters) => Promise>> - : never; - } + return output; + }, spanArgs)) as F; } /** @@ -6386,36 +6255,29 @@ export const traceable = wrapTraced; * * See {@link traced} for full details. */ -export function startSpan( - args?: StartSpanArgs & AsyncFlushArg & OptionalStateArg, -): Span { - return startSpanAndIsLogger(args).span; +export function startSpan(args?: StartSpanArgs & OptionalStateArg): Span { + return startSpanImpl(args); } /** @internal Start a span whose initial row is merged with concurrent writes. */ -export function _internalStartSpanWithInitialMerge< - IsAsyncFlush extends boolean = true, ->(args?: StartSpanArgs & AsyncFlushArg & OptionalStateArg): Span { - return startSpanAndIsLogger({ +export function _internalStartSpanWithInitialMerge( + args?: StartSpanArgs & OptionalStateArg, +): Span { + return startSpanImpl({ ...args, [INITIAL_SPAN_WRITE_AS_MERGE]: true, - } as StartSpanArgs & - AsyncFlushArg & - OptionalStateArg & - InitialSpanWriteAsMergeArg).span; + } as StartSpanArgs & OptionalStateArg & InitialSpanWriteAsMergeArg); } /** @internal Start a span with SDK-controlled context fields. */ -export function _internalStartSpanWithContext< - IsAsyncFlush extends boolean = true, ->( - args: StartSpanArgs & AsyncFlushArg & OptionalStateArg, +export function _internalStartSpanWithContext( + args: StartSpanArgs & OptionalStateArg, context: Record, ): Span { - return startSpanAndIsLogger({ + return startSpanImpl({ ...args, [INTERNAL_SPAN_CONTEXT]: context, - }).span; + }); } /** @@ -6436,23 +6298,16 @@ export function setFetch(fetch: typeof globalThis.fetch): void { _internalGetGlobalState().setFetch(fetch); } -function startSpanAndIsLogger( - args?: StartSpanArgs & - AsyncFlushArg & - OptionalStateArg & - InternalSpanContextArg, -): { span: Span; isSyncFlushLogger: boolean } { +function startSpanImpl( + args?: StartSpanArgs & OptionalStateArg & InternalSpanContextArg, +): Span { const state = args?.state ?? _globalState; // Resolve the parent object and any forwarded W3C state in one pass, so we // don't re-normalize `parent` (which could disagree if the active // logger/experiment changed between calls). const { parentObject, propagatedState } = - getSpanParentObjectAndPropagatedState({ - asyncFlush: args?.asyncFlush, - parent: args?.parent, - state, - }); + getSpanParentObjectAndPropagatedState(args); if ( parentObject instanceof SpanComponentsV3 || @@ -6472,7 +6327,7 @@ function startSpanAndIsLogger( // The parent object/state are already resolved from `parent` above; drop // the raw `parent` so it isn't re-normalized. const { parent: _ignoredParent, ...spanArgs } = args ?? {}; - const span = new SpanImpl({ + return new SpanImpl({ state, ...spanArgs, parentObjectType: parentObject.data.object_type, @@ -6490,21 +6345,8 @@ function startSpanAndIsLogger( | undefined), propagatedState, }); - return { - span, - isSyncFlushLogger: - parentObject.data.object_type === SpanObjectTypeV3.PROJECT_LOGS && - // Since there's no parent logger here, we're free to choose the async flush - // behavior, and therefore propagate along whatever we get from the arguments - args?.asyncFlush === false, - }; } else { - const span = parentObject.startSpan(args); - return { - span, - isSyncFlushLogger: - parentObject.kind === "logger" && parentObject.asyncFlush === false, - }; + return parentObject.startSpan(args); } } @@ -7197,7 +7039,7 @@ export class Experiment this.lazyMetadata = lazyMetadata; this.dataset = dataset; this.lastStartTime = getCurrentUnixTimestamp(); - this.lazyId = new LazyValue(async () => await this.id); + this.lazyId = new LazyValue(() => this.id); this.calledStartSpan = false; this.state = state; } @@ -7298,7 +7140,7 @@ export class Experiment const { setCurrent, ...argsRest } = args ?? {}; const span = this.startSpan(argsRest); - const ret = runCatchFinally( + return runCatchFinally( () => { if (setCurrent ?? true) { return withCurrent(span, callback); @@ -7306,14 +7148,12 @@ export class Experiment return callback(span); } }, - (e) => { - logError(span, e); - throw e; + (error) => { + logError(span, error); + throw error; }, () => span.end(), ); - - return ret as R; } /** @@ -7941,9 +7781,9 @@ export class SpanImpl implements Span { return callback(span); } }, - (e) => { - logError(span, e); - throw e; + (error) => { + logError(span, error); + throw error; }, () => span.end(), ); diff --git a/js/src/vendor-sdk-types/langchain.ts b/js/src/vendor-sdk-types/langchain.ts index a4ffdc909..fd795b8b5 100644 --- a/js/src/vendor-sdk-types/langchain.ts +++ b/js/src/vendor-sdk-types/langchain.ts @@ -27,10 +27,10 @@ export type LangChainCallbackManagerConfigureArgs = [ options?: unknown, ]; -export type LangChainCallbackHandlerOptions = { +export type LangChainCallbackHandlerOptions = { debug: boolean; excludeMetadataProps: RegExp; - logger?: Logger | Span; + logger?: Logger | Span; parent?: Span | (() => Span); }; diff --git a/js/src/wrappers/langchain/callback-handler.ts b/js/src/wrappers/langchain/callback-handler.ts index a0143b338..51b073173 100644 --- a/js/src/wrappers/langchain/callback-handler.ts +++ b/js/src/wrappers/langchain/callback-handler.ts @@ -15,22 +15,18 @@ import type { export const BRAINTRUST_LANGCHAIN_CALLBACK_HANDLER_NAME = "BraintrustCallbackHandler"; -export class BraintrustLangChainCallbackHandler< - IsAsyncFlush extends boolean = true, -> { +export class BraintrustLangChainCallbackHandler { name = BRAINTRUST_LANGCHAIN_CALLBACK_HANDLER_NAME; private spans = new Map(); private skippedRuns = new Set(); private parent?: Span | (() => Span); private rootRunId?: string; - private options: LangChainCallbackHandlerOptions; + private options: LangChainCallbackHandlerOptions; private startTimes = new Map(); private firstTokenTimes = new Map(); private ttftMs = new Map(); - constructor( - options?: Partial>, - ) { + constructor(options?: Partial) { this.parent = options?.parent; this.options = { debug: options?.debug ?? false, diff --git a/js/src/wrappers/test_oai_attachments.test.ts b/js/src/wrappers/test_oai_attachments.test.ts index 6fcc0ec72..443086900 100644 --- a/js/src/wrappers/test_oai_attachments.test.ts +++ b/js/src/wrappers/test_oai_attachments.test.ts @@ -11,7 +11,6 @@ describe("OpenAI attachment processing", () => { initLogger({ projectName: PROJECT_NAME, apiKey: "test-key", - asyncFlush: false, }); }); diff --git a/js/tests/api-compatibility/api-compatibility.test.ts b/js/tests/api-compatibility/api-compatibility.test.ts index 2fd785357..3941302be 100644 --- a/js/tests/api-compatibility/api-compatibility.test.ts +++ b/js/tests/api-compatibility/api-compatibility.test.ts @@ -2223,8 +2223,8 @@ describe("isUnionTypeWidening", () => { describe("areInterfaceSignaturesCompatible", () => { test("should allow adding optional fields to interface", () => { - const oldInterface = `export interface LogOptions { asyncFlush?: IsAsyncFlush; computeMetadataArgs?: Record; }`; - const newInterface = `export interface LogOptions { asyncFlush?: IsAsyncFlush; computeMetadataArgs?: Record; linkArgs?: LinkArgs; }`; + const oldInterface = `export interface FeatureOptions { enabled?: Enabled; metadata?: Record; }`; + const newInterface = `export interface FeatureOptions { enabled?: Enabled; metadata?: Record; link?: string; }`; const result = areInterfaceSignaturesCompatible(oldInterface, newInterface); expect(result).toBe(true); @@ -2239,40 +2239,40 @@ describe("areInterfaceSignaturesCompatible", () => { }); test("should reject removing fields from interface", () => { - const oldInterface = `export interface LogOptions { asyncFlush?: IsAsyncFlush; computeMetadataArgs?: Record; }`; - const newInterface = `export interface LogOptions { asyncFlush?: IsAsyncFlush; }`; + const oldInterface = `export interface FeatureOptions { enabled?: Enabled; metadata?: Record; }`; + const newInterface = `export interface FeatureOptions { enabled?: Enabled; }`; const result = areInterfaceSignaturesCompatible(oldInterface, newInterface); expect(result).toBe(false); }); test("should reject adding required fields to interface", () => { - const oldInterface = `export interface LogOptions { asyncFlush?: IsAsyncFlush; }`; - const newInterface = `export interface LogOptions { asyncFlush?: IsAsyncFlush; requiredField: string; }`; + const oldInterface = `export interface FeatureOptions { enabled?: Enabled; }`; + const newInterface = `export interface FeatureOptions { enabled?: Enabled; requiredField: string; }`; const result = areInterfaceSignaturesCompatible(oldInterface, newInterface); expect(result).toBe(false); }); test("should reject changing field types in interface", () => { - const oldInterface = `export interface LogOptions { asyncFlush?: IsAsyncFlush; computeMetadataArgs?: Record; }`; - const newInterface = `export interface LogOptions { asyncFlush?: IsAsyncFlush; computeMetadataArgs?: string; }`; + const oldInterface = `export interface FeatureOptions { enabled?: Enabled; metadata?: Record; }`; + const newInterface = `export interface FeatureOptions { enabled?: Enabled; metadata?: string; }`; const result = areInterfaceSignaturesCompatible(oldInterface, newInterface); expect(result).toBe(false); }); test("should allow making required field optional", () => { - const oldInterface = `export interface LogOptions { asyncFlush: IsAsyncFlush; }`; - const newInterface = `export interface LogOptions { asyncFlush?: IsAsyncFlush; }`; + const oldInterface = `export interface FeatureOptions { enabled: Enabled; }`; + const newInterface = `export interface FeatureOptions { enabled?: Enabled; }`; const result = areInterfaceSignaturesCompatible(oldInterface, newInterface); expect(result).toBe(true); }); test("should reject making optional field required", () => { - const oldInterface = `export interface LogOptions { asyncFlush?: IsAsyncFlush; }`; - const newInterface = `export interface LogOptions { asyncFlush: IsAsyncFlush; }`; + const oldInterface = `export interface FeatureOptions { enabled?: Enabled; }`; + const newInterface = `export interface FeatureOptions { enabled: Enabled; }`; const result = areInterfaceSignaturesCompatible(oldInterface, newInterface); expect(result).toBe(false); @@ -2577,8 +2577,8 @@ describe("areFunctionSignaturesCompatible", () => { test("should allow widening a field inside an options-object parameter", () => { // Real-world case: getSpanParentObject's options param widens its nested // `parent?` field from `string` to `string | PropagationContext`. - const oldFn = `export function getSpanParentObject(options?: AsyncFlushArg & OptionalStateArg & { parent?: string; }): Span`; - const newFn = `export function getSpanParentObject(options?: AsyncFlushArg & OptionalStateArg & { parent?: string | PropagationContext; }): Span`; + const oldFn = `export function getSpanParentObject(options?: ModeArg & OptionalStateArg & { parent?: string; }): Span`; + const newFn = `export function getSpanParentObject(options?: ModeArg & OptionalStateArg & { parent?: string | PropagationContext; }): Span`; expect(areFunctionSignaturesCompatible(oldFn, newFn)).toBe(true); }); From 51bef8cece4ee9f0ddfe1cfa626d778354ff2e93 Mon Sep 17 00:00:00 2001 From: Luca Forstner Date: Fri, 4 Sep 2026 13:58:50 +0200 Subject: [PATCH 8/8] ref!: Reduce API to a minimum that makes sense (#2429) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ref [braintrustdata/braintrust-sdk-javascript#1841](https://github.com/braintrustdata/braintrust-sdk-javascript/issues/1841) - Replaced the root `braintrust` export surface with an explicit allowlist, removing roughly 140 named exports—primarily internal helpers, implementation classes, schemas, constants, and low-level types. - Stopped named-exporting most SDK-owned request/result types. Retained types are primarily user extension points, including `StartSpanArgs`, `Span`, tracing context types, eval callbacks, reporters, template plugins, and durable-eval stores. - Removed package subpaths `braintrust/browser`, `braintrust/util`, and `braintrust/webpack-loader`; the util bundle is no longer built or published. - Kept the webpack loader private: `braintrust/next` now resolves its built loader directly instead of exposing it as a package entrypoint. - Removed public implementation/testing exports such as `_exportsForTestingOnly`, `_internalIso`, `_internalGetGlobalState`, `_internalSetInitialState`, `SpanImpl`, `NoopSpan`, `TestBackgroundLogger`, `ObjectFetcher`, and ID-generator classes. - Removed public runtime schemas and serialization internals, including prompt, stream-chunk, attachment-reference, logs-overflow, and propagation schemas/constants. - Removed lower-level project/building APIs such as `graph`, `Project`, `CodeFunction`, `CodePrompt`, `PromptBuilder`, `ScorerBuilder`, and `ToolBuilder`; the `projects` facade remains. - Removed miscellaneous convenience and implementation exports including `log`, `summarize`, `traceable`, `newId`, `permalink`, `deepCopyEvent`, template-registry internals, span fetchers, and logs-overflow helpers. - Removed deprecated APIs: - `BraintrustMiddleware` and `wrapAISDKModel` - `wrapMastraAgent` - `withExperiment` and `withDataset` - `toAIStreamResponse` - deprecated bundler aliases `esbuildPlugin`, `rollupPlugin`, `vitePlugin`, and `webpackPlugin` - deprecated `useDiagnosticChannelCompatShim` - `Experiment.close()`, `Dataset.close()`, and `Span.close()` - `EvalHooks.meta`, `EvalResultWithSummary.toString()`, and `Score.error` - Removed legacy aliases and call shapes: - `initExperiment` - string-first `init()` and `initDataset()` overloads - dataset `useOutput` mode and the `output` record alias - deprecated event `inputs` and `datasetRecordId` fields - `wrapOpenAIv4` - Changed `EvalResultWithSummary` from an exported runtime class to a plain `{ summary, results }` result shape. - Restricted public span parenting to opaque `PropagationContext`; serialized span-slug strings remain internal plumbing. The public type is still named `StartSpanArgs`. - Added `configureContextManager()` and `ContextManager.getCurrentSpanStore()` as the supported context integration mechanism, replacing global symbols and direct state mutation. - Moved instrumentation-only APIs off the root: - `configureInstrumentation`, `InstrumentationConfig`, and Eve/Flue instrumentation remain under `braintrust/instrumentation`. - `registerOtelFlush` now lives there as the narrow OTEL hook. - Simplified OTEL integration: - Uses `configureContextManager()` instead of internal globals. - No longer overrides ID generation or V4 span serialization. - Replaced `contextFromSpanExport(string)` with `contextFromSpan(span)`. - `parentFromHeaders()` now returns opaque W3C propagation context. - Environment and `.env.braintrust` handling is self-contained and Edge-safe. - Removed the OTEL-specific exported ID generator. - Migrated Temporal propagation from private serialized span headers to standard `traceparent`, `tracestate`, and `baggage`, covering client→workflow, child workflow, and same-/cross-worker activity parenting. - Reworked `@braintrust/browser` to install a browser `ContextManager` through the public configuration API instead of mutating core isomorph internals. - Updated OpenAI Agents and the Vercel adapter to avoid `braintrust/util` and the exported `BraintrustStreamChunk`; the adapter now derives its stream chunk type from `BraintrustStream`. - Canonicalized instrumentation config keys by removing typed aliases such as `vercel`, `google`, `cursor`, `bedrock`, and `awsBedrock`; environment spellings still map to canonical keys. - Updated smoke tests, integration tests, Deno/browser coverage, and e2e scenarios to use only the curated public surface; added explicit tests for the allowed root types and published package entrypoints. --- .changeset/reduce-api-to-a-minimum.md | 10 + e2e/package.json | 1 + .../vite.config.ts | 4 +- .../vite.config.ts | 4 +- .../vite.config.ts | 4 +- .../__snapshots__/request-flow.json | 200 +++++-- .../deno-browser/__snapshots__/span-tree.json | 86 +-- .../deno-browser/__snapshots__/span-tree.txt | 58 +- e2e/scenarios/deno-browser/runner.case.ts | 11 +- e2e/scenarios/deno-browser/scenario.test.ts | 14 +- e2e/scenarios/deno-node/runner.case.ts | 9 +- .../agent/instrumentation/braintrust.ts | 3 +- .../researcher/instrumentation/braintrust.ts | 3 +- e2e/scenarios/eve-instrumentation/scenario.ts | 2 +- .../flue-instrumentation/.flue/app.mjs | 3 +- .../flue-instrumentation/scenario.v2.ts | 8 +- .../otel-compat-mixed-tracing/scenario.ts | 6 +- .../test-framework-evals-jest/runner.case.cjs | 13 +- .../scenario.ts | 5 +- integrations/browser-js/src/browser-config.ts | 60 +-- integrations/langchain-js/src/test/utils.ts | 2 +- integrations/openai-agents-js/src/index.ts | 20 +- .../src/openai-agents-integration.test.ts | 6 +- .../openai-agents-js/src/test-logger.d.ts | 12 + integrations/openai-agents-js/tsconfig.json | 2 +- .../openai-agents-js/vitest.config.ts | 11 + integrations/otel-js/README.md | 15 +- integrations/otel-js/src/context.ts | 31 +- integrations/otel-js/src/exporter.test.ts | 2 +- integrations/otel-js/src/id-gen.test.ts | 7 +- integrations/otel-js/src/index.ts | 36 +- integrations/otel-js/src/otel-compat.test.ts | 86 ++- integrations/otel-js/src/otel.test.ts | 237 ++------- integrations/otel-js/src/otel.ts | 475 ++++------------- integrations/otel-js/tests/utils.ts | 13 + integrations/otel-js/tsconfig.json | 4 +- integrations/otel-js/tsconfig.test.json | 4 +- integrations/otel-js/vitest.config.ts | 2 + integrations/temporal-js/src/interceptors.ts | 123 +---- integrations/temporal-js/src/sinks.ts | 20 +- integrations/temporal-js/src/temporal.test.ts | 35 +- integrations/temporal-js/src/utils.ts | 44 +- .../temporal-js/src/workflow-interceptors.ts | 44 +- integrations/temporal-js/tsconfig.json | 4 +- integrations/temporal-js/tsup.config.ts | 1 - .../vercel-ai-sdk/src/adapter.test.ts | 9 +- integrations/vercel-ai-sdk/src/adapter.ts | 19 +- js/README.md | 4 +- js/package.json | 18 +- .../scenarios/browser-main-package/README.md | 30 +- .../src/browser-message-test.ts | 19 +- .../tests/browser-message.test.ts | 6 +- .../tests/node-esm-import.test.mjs | 4 +- .../cloudflare-vite-hono-vite-dev/README.md | 2 +- .../README.md | 2 +- .../playwright-browser/src/browser-tests.ts | 9 - js/smoke/shared/src/helpers/register.ts | 65 ++- js/smoke/shared/src/suites/basic-logging.ts | 7 + .../shared/src/suites/import-verification.ts | 133 +++-- .../auto-instrumentations/bundler/esbuild.ts | 7 - js/src/auto-instrumentations/bundler/next.ts | 16 +- .../auto-instrumentations/bundler/plugin.ts | 14 +- .../auto-instrumentations/bundler/rollup.ts | 7 - js/src/auto-instrumentations/bundler/vite.ts | 7 - .../bundler/webpack-loader.ts | 19 +- .../auto-instrumentations/bundler/webpack.ts | 7 - js/src/auto-instrumentations/configs/all.ts | 8 +- js/src/browser/index.ts | 1 - js/src/edge-light/index.ts | 1 - js/src/edge-runtime-bootstrap.test.ts | 18 +- js/src/exports.ts | 233 +------- js/src/framework.test.ts | 6 +- js/src/framework.ts | 62 +-- js/src/framework2.ts | 16 +- js/src/functions/invoke.ts | 8 +- js/src/functions/stream.ts | 69 ++- js/src/graph-framework.ts | 499 ------------------ js/src/id-gen.ts | 9 +- js/src/instrumentation/config.ts | 22 +- .../instrumentation/core/channel-tracing.ts | 16 +- js/src/instrumentation/index.test.ts | 1 + js/src/instrumentation/index.ts | 3 + .../instrumentation-consumers.test.ts | 13 +- .../instrumentation-consumers.ts | 14 +- .../claude-agent-sdk-instrumentation.test.ts | 14 +- .../claude-agent-sdk-instrumentation.ts | 2 +- .../claude-agent-sdk-local-tool-spans.ts | 2 +- .../cloudflare-ai-chat-consumer.test.ts | 22 +- .../providers/cloudflare-ai-chat-consumer.ts | 11 +- .../cloudflare-think-instrumentation.ts | 16 +- .../cursor-sdk-instrumentation.test.ts | 1 + .../providers/cursor-sdk-instrumentation.ts | 2 +- .../providers/eve-instrumentation.test.ts | 6 +- .../providers/flue-instrumentation.test.ts | 7 +- .../providers/flue-instrumentation.ts | 7 +- .../providers/genkit-instrumentation.ts | 9 +- .../github-copilot-instrumentation.ts | 2 +- .../google-adk-instrumentation.test.ts | 11 +- .../providers/google-adk-instrumentation.ts | 9 +- .../providers/google-genai-instrumentation.ts | 9 +- .../openai-codex-instrumentation.test.ts | 1 + .../providers/openai-codex-instrumentation.ts | 8 +- .../pi-coding-agent-instrumentation.test.ts | 1 + .../pi-coding-agent-instrumentation.ts | 2 +- js/src/instrumentation/registry.ts | 2 +- js/src/logger-misc.test.ts | 6 +- js/src/logger.test.ts | 15 +- js/src/logger.ts | 399 +++----------- js/src/node/index.ts | 1 - js/src/openai-utils.ts | 6 +- js/src/package-exports.test.ts | 37 ++ js/src/prompt-schemas.ts | 46 -- js/src/propagation.test.ts | 17 +- js/src/propagation.ts | 2 +- js/src/public-types.test.ts | 135 ++--- js/src/sandbox.ts | 6 +- js/src/template/registry.ts | 2 +- js/src/trace.ts | 2 +- js/src/workerd/index.ts | 1 - .../ai-sdk/deprecated/BraintrustMiddleware.ts | 493 ----------------- .../ai-sdk/deprecated/wrapAISDKModel.ts | 362 ------------- .../wrappers/ai-sdk/harness-agent-context.ts | 7 +- js/src/wrappers/ai-sdk/index.ts | 4 - js/src/wrappers/exports.test.ts | 38 -- js/src/wrappers/langchain/callback-handler.ts | 5 +- js/src/wrappers/mastra.ts | 46 +- js/src/wrappers/node-test/suite.ts | 5 +- js/src/wrappers/node-test/test-helpers.ts | 14 +- js/src/wrappers/oai.ts | 9 +- js/src/wrappers/vitest-evals/reporter.test.ts | 2 +- js/src/wrappers/vitest-evals/reporter.ts | 9 +- js/src/wrappers/vitest/context-manager.ts | 4 +- .../vitest/vitest-wrapper-example.test.ts | 2 +- .../vitest/vitest-wrapper-span.test.ts | 2 +- js/src/wrappers/vitest/vitest-wrapper.test.ts | 27 +- js/src/wrappers/vitest/wrapper.ts | 7 +- .../api-compatibility.test.ts | 35 +- .../error-handling.test.ts | 28 +- .../event-content.test.ts | 80 +-- .../function-behavior.test.ts | 36 +- .../multiple-instrumentations.test.ts | 24 +- .../auto-instrumentations/next-config.test.ts | 10 +- .../runtime-execution.test.ts | 36 +- .../streaming-and-responses.test.ts | 24 +- .../transformation.test.ts | 10 +- js/tsup.config.ts | 31 +- js/util/object.ts | 3 +- js/util/score.ts | 5 - pnpm-lock.yaml | 3 + 149 files changed, 1406 insertions(+), 3917 deletions(-) create mode 100644 .changeset/reduce-api-to-a-minimum.md create mode 100644 integrations/openai-agents-js/src/test-logger.d.ts delete mode 100644 js/src/graph-framework.ts create mode 100644 js/src/package-exports.test.ts delete mode 100644 js/src/wrappers/ai-sdk/deprecated/BraintrustMiddleware.ts delete mode 100644 js/src/wrappers/ai-sdk/deprecated/wrapAISDKModel.ts delete mode 100644 js/src/wrappers/exports.test.ts diff --git a/.changeset/reduce-api-to-a-minimum.md b/.changeset/reduce-api-to-a-minimum.md new file mode 100644 index 000000000..958c893c9 --- /dev/null +++ b/.changeset/reduce-api-to-a-minimum.md @@ -0,0 +1,10 @@ +--- +"braintrust": major +"@braintrust/otel": major +"@braintrust/vercel-ai-sdk": major +"@braintrust/browser": patch +"@braintrust/openai-agents": patch +"@braintrust/temporal": patch +--- + +ref!: Reduce API to a minimum that makes sense diff --git a/e2e/package.json b/e2e/package.json index 8de220b7e..414b75247 100644 --- a/e2e/package.json +++ b/e2e/package.json @@ -8,6 +8,7 @@ "test:e2e:record": "node ./scripts/run-record-tests.mjs" }, "devDependencies": { + "@braintrust/browser": "workspace:^", "@braintrust/langchain-js": "workspace:^", "@braintrust/otel": "workspace:^", "@braintrust/seinfeld": "workspace:^", diff --git a/e2e/scenarios/cloudflare-agents-instrumentation/vite.config.ts b/e2e/scenarios/cloudflare-agents-instrumentation/vite.config.ts index ca68896e5..0e75a3319 100644 --- a/e2e/scenarios/cloudflare-agents-instrumentation/vite.config.ts +++ b/e2e/scenarios/cloudflare-agents-instrumentation/vite.config.ts @@ -7,9 +7,7 @@ const agentsPackageName = const instrumentationMode = process.env.CLOUDFLARE_AGENTS_INSTRUMENTATION_MODE ?? "auto"; const braintrustPlugins = - instrumentationMode === "auto" - ? braintrustVitePlugin({ useDiagnosticChannelCompatShim: true }) - : []; + instrumentationMode === "auto" ? braintrustVitePlugin({ browser: true }) : []; export default defineConfig({ define: { diff --git a/e2e/scenarios/cloudflare-ai-chat-instrumentation/vite.config.ts b/e2e/scenarios/cloudflare-ai-chat-instrumentation/vite.config.ts index 3952e5fd6..1c131a62d 100644 --- a/e2e/scenarios/cloudflare-ai-chat-instrumentation/vite.config.ts +++ b/e2e/scenarios/cloudflare-ai-chat-instrumentation/vite.config.ts @@ -25,9 +25,7 @@ export default defineConfig({ __OPENAI_BASE_URL__: JSON.stringify(process.env.OPENAI_BASE_URL), }, plugins: [ - ...(mode === "auto" - ? [braintrustVitePlugin({ useDiagnosticChannelCompatShim: true })] - : []), + ...(mode === "auto" ? [braintrustVitePlugin({ browser: true })] : []), cloudflare({ configPath: "./wrangler.toml", inspectorPort: 0 }), ], resolve: { diff --git a/e2e/scenarios/cloudflare-think-instrumentation/vite.config.ts b/e2e/scenarios/cloudflare-think-instrumentation/vite.config.ts index 9a28d8c50..d068998ef 100644 --- a/e2e/scenarios/cloudflare-think-instrumentation/vite.config.ts +++ b/e2e/scenarios/cloudflare-think-instrumentation/vite.config.ts @@ -13,9 +13,7 @@ export default defineConfig({ ), }, plugins: [ - ...(manual - ? [] - : [braintrustVitePlugin({ useDiagnosticChannelCompatShim: true })]), + ...(manual ? [] : [braintrustVitePlugin({ browser: true })]), cloudflare({ inspectorPort: 0 }), ], resolve: { diff --git a/e2e/scenarios/deno-browser/__snapshots__/request-flow.json b/e2e/scenarios/deno-browser/__snapshots__/request-flow.json index f8eea5a3f..ca74c4553 100644 --- a/e2e/scenarios/deno-browser/__snapshots__/request-flow.json +++ b/e2e/scenarios/deno-browser/__snapshots__/request-flow.json @@ -27,7 +27,11 @@ "api_version": 2, "rows": [ { - "context": {}, + "context": { + "caller_filename": "ext:cli/40_test.js", + "caller_functionname": "innerWrapped", + "caller_lineno": 0 + }, "created": "", "expected": "Paris", "id": "", @@ -54,7 +58,11 @@ "span_id": "" }, { - "context": {}, + "context": { + "caller_filename": "ext:cli/40_test.js", + "caller_functionname": "innerWrapped", + "caller_lineno": 0 + }, "created": "", "id": "", "input": { @@ -89,7 +97,11 @@ "span_id": "" }, { - "context": {}, + "context": { + "caller_filename": "ext:cli/40_test.js", + "caller_functionname": "innerWrapped", + "caller_lineno": 0 + }, "created": "", "id": "", "input": { @@ -120,7 +132,11 @@ "span_id": "" }, { - "context": {}, + "context": { + "caller_filename": "/e2e/scenarios/deno-browser/runner.case.ts", + "caller_functionname": "braintrust.traced.name", + "caller_lineno": 0 + }, "created": "", "id": "", "input": { @@ -142,18 +158,24 @@ "phase": "child" }, "project_id": "", - "root_span_id": "", + "root_span_id": "", "span_attributes": { "exec_counter": 3, - "name": "deno browser child span", - "type": "task" + "name": "deno browser child span" }, - "span_id": "" + "span_id": "", + "span_parents": [ + "" + ] }, { - "context": {}, + "context": { + "caller_filename": "ext:cli/40_test.js", + "caller_functionname": "async innerWrapped", + "caller_lineno": 0 + }, "created": "", - "id": "", + "id": "", "log_id": "g", "metadata": { "case": "nested-parent", @@ -165,18 +187,22 @@ "start": 0 }, "project_id": "", - "root_span_id": "", + "root_span_id": "", "span_attributes": { "exec_counter": 4, "name": "deno browser nested parent span", "type": "task" }, - "span_id": "" + "span_id": "" }, { - "context": {}, + "context": { + "caller_filename": "/e2e/scenarios/deno-browser/runner.case.ts", + "caller_functionname": "braintrust.traced.name", + "caller_lineno": 0 + }, "created": "", - "id": "", + "id": "", "log_id": "g", "metadata": { "case": "nested-child", @@ -188,18 +214,24 @@ "start": 0 }, "project_id": "", - "root_span_id": "", + "root_span_id": "", "span_attributes": { "exec_counter": 5, - "name": "deno browser nested child span", - "type": "task" + "name": "deno browser nested child span" }, - "span_id": "" + "span_id": "", + "span_parents": [ + "" + ] }, { - "context": {}, + "context": { + "caller_filename": "/e2e/scenarios/deno-browser/runner.case.ts", + "caller_functionname": "braintrust.traced.name", + "caller_lineno": 0 + }, "created": "", - "id": "", + "id": "", "log_id": "g", "metadata": { "case": "nested-grandchild", @@ -210,19 +242,28 @@ "end": 0, "start": 0 }, + "output": { + "depth": 3 + }, "project_id": "", - "root_span_id": "", + "root_span_id": "", "span_attributes": { "exec_counter": 6, - "name": "deno browser nested grandchild span", - "type": "task" + "name": "deno browser nested grandchild span" }, - "span_id": "" + "span_id": "", + "span_parents": [ + "" + ] }, { - "context": {}, + "context": { + "caller_filename": "ext:cli/40_test.js", + "caller_functionname": "async innerWrapped", + "caller_lineno": 0 + }, "created": "", - "id": "", + "id": "", "log_id": "g", "metadata": { "case": "current-span", @@ -233,14 +274,17 @@ "end": 0, "start": 0 }, + "output": { + "observedSpanId": "" + }, "project_id": "", - "root_span_id": "", + "root_span_id": "", "span_attributes": { "exec_counter": 7, "name": "deno browser current span", "type": "task" }, - "span_id": "" + "span_id": "" } ] }, @@ -251,7 +295,11 @@ "api_version": 2, "rows": [ { - "context": {}, + "context": { + "caller_filename": "ext:cli/40_test.js", + "caller_functionname": "innerWrapped", + "caller_lineno": 0 + }, "created": "", "expected": "Paris", "id": "", @@ -278,7 +326,11 @@ "span_id": "" }, { - "context": {}, + "context": { + "caller_filename": "ext:cli/40_test.js", + "caller_functionname": "innerWrapped", + "caller_lineno": 0 + }, "created": "", "id": "", "input": { @@ -313,7 +365,11 @@ "span_id": "" }, { - "context": {}, + "context": { + "caller_filename": "ext:cli/40_test.js", + "caller_functionname": "innerWrapped", + "caller_lineno": 0 + }, "created": "", "id": "", "input": { @@ -344,7 +400,11 @@ "span_id": "" }, { - "context": {}, + "context": { + "caller_filename": "/e2e/scenarios/deno-browser/runner.case.ts", + "caller_functionname": "braintrust.traced.name", + "caller_lineno": 0 + }, "created": "", "id": "", "input": { @@ -366,18 +426,24 @@ "phase": "child" }, "project_id": "", - "root_span_id": "", + "root_span_id": "", "span_attributes": { "exec_counter": 3, - "name": "deno browser child span", - "type": "task" + "name": "deno browser child span" }, - "span_id": "" + "span_id": "", + "span_parents": [ + "" + ] }, { - "context": {}, + "context": { + "caller_filename": "ext:cli/40_test.js", + "caller_functionname": "async innerWrapped", + "caller_lineno": 0 + }, "created": "", - "id": "", + "id": "", "log_id": "g", "metadata": { "case": "nested-parent", @@ -389,18 +455,22 @@ "start": 0 }, "project_id": "", - "root_span_id": "", + "root_span_id": "", "span_attributes": { "exec_counter": 4, "name": "deno browser nested parent span", "type": "task" }, - "span_id": "" + "span_id": "" }, { - "context": {}, + "context": { + "caller_filename": "/e2e/scenarios/deno-browser/runner.case.ts", + "caller_functionname": "braintrust.traced.name", + "caller_lineno": 0 + }, "created": "", - "id": "", + "id": "", "log_id": "g", "metadata": { "case": "nested-child", @@ -412,18 +482,24 @@ "start": 0 }, "project_id": "", - "root_span_id": "", + "root_span_id": "", "span_attributes": { "exec_counter": 5, - "name": "deno browser nested child span", - "type": "task" + "name": "deno browser nested child span" }, - "span_id": "" + "span_id": "", + "span_parents": [ + "" + ] }, { - "context": {}, + "context": { + "caller_filename": "/e2e/scenarios/deno-browser/runner.case.ts", + "caller_functionname": "braintrust.traced.name", + "caller_lineno": 0 + }, "created": "", - "id": "", + "id": "", "log_id": "g", "metadata": { "case": "nested-grandchild", @@ -434,19 +510,28 @@ "end": 0, "start": 0 }, + "output": { + "depth": 3 + }, "project_id": "", - "root_span_id": "", + "root_span_id": "", "span_attributes": { "exec_counter": 6, - "name": "deno browser nested grandchild span", - "type": "task" + "name": "deno browser nested grandchild span" }, - "span_id": "" + "span_id": "", + "span_parents": [ + "" + ] }, { - "context": {}, + "context": { + "caller_filename": "ext:cli/40_test.js", + "caller_functionname": "async innerWrapped", + "caller_lineno": 0 + }, "created": "", - "id": "", + "id": "", "log_id": "g", "metadata": { "case": "current-span", @@ -457,14 +542,17 @@ "end": 0, "start": 0 }, + "output": { + "observedSpanId": "" + }, "project_id": "", - "root_span_id": "", + "root_span_id": "", "span_attributes": { "exec_counter": 7, "name": "deno browser current span", "type": "task" }, - "span_id": "" + "span_id": "" } ] } diff --git a/e2e/scenarios/deno-browser/__snapshots__/span-tree.json b/e2e/scenarios/deno-browser/__snapshots__/span-tree.json index 9461d0edb..d5ffa8e80 100644 --- a/e2e/scenarios/deno-browser/__snapshots__/span-tree.json +++ b/e2e/scenarios/deno-browser/__snapshots__/span-tree.json @@ -39,7 +39,25 @@ { "name": "deno browser parent span", "type": "task", - "children": [], + "children": [ + { + "name": "deno browser child span", + "children": [], + "input": { + "step": "child", + "testRunId": "" + }, + "output": { + "ok": true, + "phase": "child" + }, + "metadata": { + "case": "child-span", + "scenario": "deno-browser", + "testRunId": "" + } + } + ], "input": { "phase": "parent", "testRunId": "" @@ -54,58 +72,46 @@ "testRunId": "" } }, - { - "name": "deno browser child span", - "type": "task", - "children": [], - "input": { - "step": "child", - "testRunId": "" - }, - "output": { - "ok": true, - "phase": "child" - }, - "metadata": { - "case": "child-span", - "scenario": "deno-browser", - "testRunId": "" - } - }, { "name": "deno browser nested parent span", "type": "task", - "children": [], + "children": [ + { + "name": "deno browser nested child span", + "children": [ + { + "name": "deno browser nested grandchild span", + "children": [], + "output": { + "depth": 3 + }, + "metadata": { + "case": "nested-grandchild", + "scenario": "deno-browser", + "testRunId": "" + } + } + ], + "metadata": { + "case": "nested-child", + "scenario": "deno-browser", + "testRunId": "" + } + } + ], "metadata": { "case": "nested-parent", "scenario": "deno-browser", "testRunId": "" } }, - { - "name": "deno browser nested child span", - "type": "task", - "children": [], - "metadata": { - "case": "nested-child", - "scenario": "deno-browser", - "testRunId": "" - } - }, - { - "name": "deno browser nested grandchild span", - "type": "task", - "children": [], - "metadata": { - "case": "nested-grandchild", - "scenario": "deno-browser", - "testRunId": "" - } - }, { "name": "deno browser current span", "type": "task", "children": [], + "output": { + "observedSpanId": "" + }, "metadata": { "case": "current-span", "scenario": "deno-browser", diff --git a/e2e/scenarios/deno-browser/__snapshots__/span-tree.txt b/e2e/scenarios/deno-browser/__snapshots__/span-tree.txt index 9af1c12e1..3f459283e 100644 --- a/e2e/scenarios/deno-browser/__snapshots__/span-tree.txt +++ b/e2e/scenarios/deno-browser/__snapshots__/span-tree.txt @@ -41,39 +41,45 @@ span_tree: │ "scenario": "deno-browser", │ "testRunId": "" │ } -├── deno browser child span [task] -│ input: { -│ "step": "child", -│ "testRunId": "" -│ } -│ output: { -│ "ok": true, -│ "phase": "child" -│ } -│ metadata: { -│ "case": "child-span", -│ "scenario": "deno-browser", -│ "testRunId": "" -│ } +│ └── deno browser child span +│ input: { +│ "step": "child", +│ "testRunId": "" +│ } +│ output: { +│ "ok": true, +│ "phase": "child" +│ } +│ metadata: { +│ "case": "child-span", +│ "scenario": "deno-browser", +│ "testRunId": "" +│ } ├── deno browser nested parent span [task] │ metadata: { │ "case": "nested-parent", │ "scenario": "deno-browser", │ "testRunId": "" │ } -├── deno browser nested child span [task] -│ metadata: { -│ "case": "nested-child", -│ "scenario": "deno-browser", -│ "testRunId": "" -│ } -├── deno browser nested grandchild span [task] -│ metadata: { -│ "case": "nested-grandchild", -│ "scenario": "deno-browser", -│ "testRunId": "" -│ } +│ └── deno browser nested child span +│ metadata: { +│ "case": "nested-child", +│ "scenario": "deno-browser", +│ "testRunId": "" +│ } +│ └── deno browser nested grandchild span +│ output: { +│ "depth": 3 +│ } +│ metadata: { +│ "case": "nested-grandchild", +│ "scenario": "deno-browser", +│ "testRunId": "" +│ } └── deno browser current span [task] + output: { + "observedSpanId": "" + } metadata: { "case": "current-span", "scenario": "deno-browser", diff --git a/e2e/scenarios/deno-browser/runner.case.ts b/e2e/scenarios/deno-browser/runner.case.ts index 88197b73b..823b53dc6 100644 --- a/e2e/scenarios/deno-browser/runner.case.ts +++ b/e2e/scenarios/deno-browser/runner.case.ts @@ -1,6 +1,5 @@ -import * as braintrust from "braintrust/browser"; +import * as braintrust from "@braintrust/browser"; import { - expectBuildType, expectEvalWorks, expectMustacheTemplate, expectNamedExports, @@ -27,10 +26,8 @@ Deno.test("deno-browser exposes the expected browser runtime surface", () => { "initLogger", "currentSpan", "startSpan", - "log", "flush", "initDataset", - "initExperiment", "loadPrompt", "Prompt", "Eval", @@ -40,14 +37,10 @@ Deno.test("deno-browser exposes the expected browser runtime surface", () => { "Attachment", "invoke", "initFunction", - "Project", - "PromptBuilder", - "IDGenerator", + "projects", "BraintrustState", "login", - "_exportsForTestingOnly", ]); - expectBuildType(braintrust, "browser"); expectMustacheTemplate(braintrust); expectNunjucksTemplateUnavailable(braintrust); }); diff --git a/e2e/scenarios/deno-browser/scenario.test.ts b/e2e/scenarios/deno-browser/scenario.test.ts index 46fe46f7d..eb066f2d1 100644 --- a/e2e/scenarios/deno-browser/scenario.test.ts +++ b/e2e/scenarios/deno-browser/scenario.test.ts @@ -128,11 +128,15 @@ test( phase: "child", ok: true, }); - expect(childSpan?.span.parentIds).toEqual([]); - expect(nestedChild?.span.parentIds).toEqual([]); - expect(nestedGrandchild?.span.parentIds).toEqual([]); - expect(nestedGrandchild?.output).toBeUndefined(); - expect(currentSpan?.output).toBeUndefined(); + expect(childSpan?.span.parentIds).toEqual([parentSpan?.span.id]); + expect(nestedChild?.span.parentIds).toEqual([nestedParent?.span.id]); + expect(nestedGrandchild?.span.parentIds).toEqual([ + nestedChild?.span.id, + ]); + expect(nestedGrandchild?.output).toEqual({ depth: 3 }); + expect(currentSpan?.output).toMatchObject({ + observedSpanId: currentSpan?.span.id, + }); const requests = requestsAfter( cursor, diff --git a/e2e/scenarios/deno-node/runner.case.ts b/e2e/scenarios/deno-node/runner.case.ts index b8f88b2db..2f21d1d91 100644 --- a/e2e/scenarios/deno-node/runner.case.ts +++ b/e2e/scenarios/deno-node/runner.case.ts @@ -1,6 +1,5 @@ import * as braintrust from "braintrust"; import { - expectBuildType, expectEvalWorks, expectMustacheTemplate, expectNamedExports, @@ -27,10 +26,8 @@ Deno.test("deno-node exposes the expected node runtime surface", () => { "initLogger", "currentSpan", "startSpan", - "log", "flush", "initDataset", - "initExperiment", "loadPrompt", "Prompt", "Eval", @@ -40,14 +37,10 @@ Deno.test("deno-node exposes the expected node runtime surface", () => { "Attachment", "invoke", "initFunction", - "Project", - "PromptBuilder", - "IDGenerator", + "projects", "BraintrustState", "login", - "_exportsForTestingOnly", ]); - expectBuildType(braintrust, "node"); expectMustacheTemplate(braintrust); expectNunjucksTemplateUnavailable(braintrust); }); diff --git a/e2e/scenarios/eve-instrumentation/agent/instrumentation/braintrust.ts b/e2e/scenarios/eve-instrumentation/agent/instrumentation/braintrust.ts index 9ba5484f2..b16977e74 100644 --- a/e2e/scenarios/eve-instrumentation/agent/instrumentation/braintrust.ts +++ b/e2e/scenarios/eve-instrumentation/agent/instrumentation/braintrust.ts @@ -1,4 +1,5 @@ -import { braintrustEveInstrumentation, initLogger } from "braintrust"; +import { initLogger } from "braintrust"; +import { braintrustEveInstrumentation } from "braintrust/instrumentation"; import { defineInstrumentation } from "eve/instrumentation"; export default defineInstrumentation( diff --git a/e2e/scenarios/eve-instrumentation/agent/subagents/researcher/instrumentation/braintrust.ts b/e2e/scenarios/eve-instrumentation/agent/subagents/researcher/instrumentation/braintrust.ts index 9ba5484f2..b16977e74 100644 --- a/e2e/scenarios/eve-instrumentation/agent/subagents/researcher/instrumentation/braintrust.ts +++ b/e2e/scenarios/eve-instrumentation/agent/subagents/researcher/instrumentation/braintrust.ts @@ -1,4 +1,5 @@ -import { braintrustEveInstrumentation, initLogger } from "braintrust"; +import { initLogger } from "braintrust"; +import { braintrustEveInstrumentation } from "braintrust/instrumentation"; import { defineInstrumentation } from "eve/instrumentation"; export default defineInstrumentation( diff --git a/e2e/scenarios/eve-instrumentation/scenario.ts b/e2e/scenarios/eve-instrumentation/scenario.ts index ca1c92027..446677607 100644 --- a/e2e/scenarios/eve-instrumentation/scenario.ts +++ b/e2e/scenarios/eve-instrumentation/scenario.ts @@ -80,7 +80,7 @@ async function main() { ); // Eve emits session.waiting just before its durable session snapshot is // visible to the continuation route. - await new Promise((resolve) => setTimeout(resolve, 1000)); + await new Promise((resolve) => setTimeout(resolve, 3000)); const followUp = await fetch( `${baseUrl}/eve/v1/session/${body.sessionId}`, { diff --git a/e2e/scenarios/flue-instrumentation/.flue/app.mjs b/e2e/scenarios/flue-instrumentation/.flue/app.mjs index 5d3ce1360..7e22bc269 100644 --- a/e2e/scenarios/flue-instrumentation/.flue/app.mjs +++ b/e2e/scenarios/flue-instrumentation/.flue/app.mjs @@ -1,4 +1,5 @@ -import { braintrustFlueInstrumentation, flush, initLogger } from "braintrust"; +import { flush, initLogger } from "braintrust"; +import { braintrustFlueInstrumentation } from "braintrust/instrumentation"; import { Hono } from "hono"; const runtimePackageName = diff --git a/e2e/scenarios/flue-instrumentation/scenario.v2.ts b/e2e/scenarios/flue-instrumentation/scenario.v2.ts index 567d47321..8d9c41a06 100644 --- a/e2e/scenarios/flue-instrumentation/scenario.v2.ts +++ b/e2e/scenarios/flue-instrumentation/scenario.v2.ts @@ -1,10 +1,6 @@ import { openaiProvider } from "pi-ai-v2/providers/openai"; -import { - braintrustFlueInstrumentation, - flush, - initLogger, - traced, -} from "braintrust"; +import { flush, initLogger, traced } from "braintrust"; +import { braintrustFlueInstrumentation } from "braintrust/instrumentation"; import * as v from "valibot"; const SCENARIO_NAME = "flue-instrumentation"; diff --git a/e2e/scenarios/otel-compat-mixed-tracing/scenario.ts b/e2e/scenarios/otel-compat-mixed-tracing/scenario.ts index f3a2d4bb1..f74828b5e 100644 --- a/e2e/scenarios/otel-compat-mixed-tracing/scenario.ts +++ b/e2e/scenarios/otel-compat-mixed-tracing/scenario.ts @@ -1,7 +1,7 @@ import { context as otelContext, trace } from "@opentelemetry/api"; import { AsyncHooksContextManager } from "@opentelemetry/context-async-hooks"; import { BraintrustSpanProcessor, setupOtelCompat } from "@braintrust/otel"; -import { getContextManager, initLogger } from "braintrust"; +import { initLogger, withCurrent } from "braintrust"; import { createTracerProvider, getTestRunId, @@ -39,9 +39,7 @@ async function main() { }, }, }); - const contextManagerFacade = getContextManager(); - - await contextManagerFacade.runInContext(btRoot, async () => { + await withCurrent(btRoot, async () => { await tracer.startActiveSpan("otel-middle", async (otelSpan) => { const btChild = logger.startSpan({ name: "bt-child-under-otel", diff --git a/e2e/scenarios/test-framework-evals-jest/runner.case.cjs b/e2e/scenarios/test-framework-evals-jest/runner.case.cjs index cffa7756d..b2886017b 100644 --- a/e2e/scenarios/test-framework-evals-jest/runner.case.cjs +++ b/e2e/scenarios/test-framework-evals-jest/runner.case.cjs @@ -2,7 +2,6 @@ const { Eval, JSONAttachment, Prompt, - _exportsForTestingOnly, currentSpan, initLogger, startSpan, @@ -40,10 +39,8 @@ test("jest exposes the core braintrust runtime surface in CommonJS", () => { "initLogger", "currentSpan", "startSpan", - "log", "flush", "initDataset", - "initExperiment", "loadPrompt", "Prompt", "Eval", @@ -53,20 +50,12 @@ test("jest exposes the core braintrust runtime surface in CommonJS", () => { "Attachment", "invoke", "initFunction", - "Project", - "PromptBuilder", - "IDGenerator", + "projects", "BraintrustState", "login", - "_exportsForTestingOnly", ]); }); -test("jest resolves the node build in CommonJS mode", () => { - expect(_exportsForTestingOnly).toBeDefined(); - expect(_exportsForTestingOnly.isomorph.buildType).toBe("node"); -}); - test("jest renders mustache prompts and fails clearly for nunjucks without the extra package", () => { const mustachePrompt = new Prompt( { diff --git a/e2e/scenarios/trace-context-and-continuation/scenario.ts b/e2e/scenarios/trace-context-and-continuation/scenario.ts index c0bb88ec3..bcf1371ba 100644 --- a/e2e/scenarios/trace-context-and-continuation/scenario.ts +++ b/e2e/scenarios/trace-context-and-continuation/scenario.ts @@ -1,5 +1,6 @@ import { flush, + extractTraceContextFromHeaders, initLogger, startSpan, traced, @@ -28,7 +29,7 @@ async function main() { }, }, }); - const exportedRoot = await rootSpan.export(); + const propagatedRoot = extractTraceContextFromHeaders(rootSpan.inject()); await withCurrent(rootSpan, async () => { const currentChild = startSpan({ @@ -50,7 +51,7 @@ async function main() { rootSpan.end(); - await withParent(exportedRoot, async () => { + await withParent(propagatedRoot, async () => { await traced( (span) => { span.log({ diff --git a/integrations/browser-js/src/browser-config.ts b/integrations/browser-js/src/browser-config.ts index ddf239c33..ae9c788a6 100644 --- a/integrations/browser-js/src/browser-config.ts +++ b/integrations/browser-js/src/browser-config.ts @@ -1,41 +1,35 @@ -import { _internalIso as iso, _internalSetInitialState } from "braintrust"; +import { + configureContextManager, + ContextManager, + type ContextParentSpanIds, + type CurrentSpanStore, + type Span, +} from "braintrust"; import { AsyncLocalStorage as BrowserAsyncLocalStorage } from "als-browser"; -export function configureBrowser() { - // Set build type indicator - iso.buildType = "browser-js" as "browser"; +class BrowserContextManager extends ContextManager { + private readonly currentSpan = new BrowserAsyncLocalStorage(); - iso.newAsyncLocalStorage = () => new BrowserAsyncLocalStorage(); + getParentSpanIds(): ContextParentSpanIds | undefined { + const span = this.currentSpan.getStore(); + return span + ? { rootSpanId: span.rootSpanId, spanParents: [span.spanId] } + : undefined; + } - iso.getEnv = (name: string) => { - if (typeof process === "undefined" || typeof process.env === "undefined") { - return undefined; - } - return process.env[name]; - }; + runInContext(span: Span, callback: () => R): R { + return this.currentSpan.run(span, callback); + } - // noop implementations for git config - iso.getRepoInfo = async () => ({ - commit: null, - branch: null, - tag: null, - dirty: false, - }); - iso.getCallerLocation = () => undefined; + getCurrentSpan(): Span | undefined { + return this.currentSpan.getStore(); + } - // Implement browser-compatible hash function using a simple hash algorithm - iso.hash = (data: string): string => { - // Simple hash function for browser compatibility - let hash = 0; - for (let i = 0; i < data.length; i++) { - const char = data.charCodeAt(i); - hash = (hash << 5) - hash + char; - hash = hash & hash; // Convert to 32bit integer - } - // Convert to hex string - const hashHex = (hash >>> 0).toString(16).padStart(8, "0"); - return hashHex.repeat(8).substring(0, 64); // Make it look like a SHA-256 hash length - }; + getCurrentSpanStore(): CurrentSpanStore { + return this.currentSpan; + } +} - _internalSetInitialState(); +export function configureBrowser(): void { + configureContextManager(BrowserContextManager); } diff --git a/integrations/langchain-js/src/test/utils.ts b/integrations/langchain-js/src/test/utils.ts index 823dc5b71..52761197c 100644 --- a/integrations/langchain-js/src/test/utils.ts +++ b/integrations/langchain-js/src/test/utils.ts @@ -1,4 +1,4 @@ -import { mergeDicts } from "braintrust/util"; +import { mergeDicts } from "../../../../js/util/object_util"; import { BaseCallbackHandler } from "@langchain/core/callbacks/base"; import { LogsRequest } from "./types"; diff --git a/integrations/openai-agents-js/src/index.ts b/integrations/openai-agents-js/src/index.ts index beafc3fc0..233bfa9bb 100644 --- a/integrations/openai-agents-js/src/index.ts +++ b/integrations/openai-agents-js/src/index.ts @@ -1,5 +1,4 @@ /* eslint-disable @typescript-eslint/no-explicit-any */ -import { SpanTypeAttribute } from "braintrust/util"; import { Span as BraintrustSpan, startSpan, @@ -7,6 +6,7 @@ import { currentSpan, NOOP_SPAN, Attachment, + type StartSpanArgs, } from "braintrust"; import { SpanType, @@ -29,7 +29,9 @@ import { isSpeechGroupSpanData, } from "./types"; -function spanTypeFromAgents(span: AgentsSpan): SpanTypeAttribute { +function spanTypeFromAgents( + span: AgentsSpan, +): NonNullable { const spanType = span.spanData.type; if ( @@ -38,7 +40,7 @@ function spanTypeFromAgents(span: AgentsSpan): SpanTypeAttribute { spanType === SpanType.CUSTOM || spanType === SpanType.SPEECH_GROUP ) { - return SpanTypeAttribute.TASK; + return "task"; } if ( @@ -46,7 +48,7 @@ function spanTypeFromAgents(span: AgentsSpan): SpanTypeAttribute { spanType === SpanType.GUARDRAIL || spanType === SpanType.MCP_TOOLS ) { - return SpanTypeAttribute.TOOL; + return "tool"; } if ( @@ -55,10 +57,10 @@ function spanTypeFromAgents(span: AgentsSpan): SpanTypeAttribute { spanType === SpanType.TRANSCRIPTION || spanType === SpanType.SPEECH ) { - return SpanTypeAttribute.LLM; + return "llm"; } - return SpanTypeAttribute.TASK; + return "task"; } function spanNameFromAgents(span: AgentsSpan): string { @@ -282,18 +284,18 @@ export class OpenAIAgentsTraceProcessor { // Create as child of current span span = current.startSpan({ name: trace.name, - type: SpanTypeAttribute.TASK, + type: "task", }); } else { // No parent span available, create as root span = this.logger ? this.logger.startSpan({ name: trace.name, - type: SpanTypeAttribute.TASK, + type: "task", }) : startSpan({ name: trace.name, - type: SpanTypeAttribute.TASK, + type: "task", }); } diff --git a/integrations/openai-agents-js/src/openai-agents-integration.test.ts b/integrations/openai-agents-js/src/openai-agents-integration.test.ts index 71a6e5fd5..e6f59c0cd 100644 --- a/integrations/openai-agents-js/src/openai-agents-integration.test.ts +++ b/integrations/openai-agents-js/src/openai-agents-integration.test.ts @@ -11,14 +11,16 @@ import { z } from "zod/v3"; // Import necessary types and functions from braintrust import { - _exportsForTestingOnly, initLogger, Logger, - TestBackgroundLogger, Span as BraintrustSpan, wrapTraced, currentSpan, } from "braintrust"; +import { + _exportsForTestingOnly, + type TestBackgroundLogger, +} from "@braintrust-test/logger"; // Test helper functions for backward compatibility function getSpansMap( diff --git a/integrations/openai-agents-js/src/test-logger.d.ts b/integrations/openai-agents-js/src/test-logger.d.ts new file mode 100644 index 000000000..1cc19256d --- /dev/null +++ b/integrations/openai-agents-js/src/test-logger.d.ts @@ -0,0 +1,12 @@ +declare module "@braintrust-test/logger" { + export interface TestBackgroundLogger { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + drain(): Promise; + } + + export const _exportsForTestingOnly: { + simulateLoginForTests(): Promise; + useTestBackgroundLogger(): TestBackgroundLogger; + clearTestBackgroundLogger(): void; + }; +} diff --git a/integrations/openai-agents-js/tsconfig.json b/integrations/openai-agents-js/tsconfig.json index ea6ad1861..5fd07c79d 100644 --- a/integrations/openai-agents-js/tsconfig.json +++ b/integrations/openai-agents-js/tsconfig.json @@ -10,6 +10,6 @@ "esModuleInterop": true, "skipLibCheck": true }, - "include": ["."], + "include": ["src"], "exclude": ["node_modules/**", "**/dist/**"] } diff --git a/integrations/openai-agents-js/vitest.config.ts b/integrations/openai-agents-js/vitest.config.ts index 9b3c6bb43..1c14e5ef6 100644 --- a/integrations/openai-agents-js/vitest.config.ts +++ b/integrations/openai-agents-js/vitest.config.ts @@ -1,6 +1,17 @@ import { defineConfig } from "vitest/config"; +import { fileURLToPath } from "node:url"; export default defineConfig({ + resolve: { + alias: { + "@braintrust-test/logger": fileURLToPath( + new URL("../../js/src/logger.ts", import.meta.url), + ), + braintrust: fileURLToPath( + new URL("../../js/src/node/index.ts", import.meta.url), + ), + }, + }, test: { reporters: ["default"], include: ["tests/**/*.test.ts", "src/**/*.test.ts"], diff --git a/integrations/otel-js/README.md b/integrations/otel-js/README.md index f5adf45c2..e7afa2da5 100644 --- a/integrations/otel-js/README.md +++ b/integrations/otel-js/README.md @@ -118,28 +118,23 @@ const logger = initLogger({ projectName: "my_project" }); **What it does**: - Sets Braintrust to use OpenTelemetry's context manager for parent-child span relationships -- Configures Braintrust to use OpenTelemetry-compatible span and trace IDs - Enables seamless mixing of Braintrust and OpenTelemetry spans in the same trace ## Distributed Tracing Helpers These utilities enable distributed tracing across services that use different tracing systems (Braintrust and OpenTelemetry). -### `contextFromSpanExport(exportStr: string)` +### `contextFromSpan(span: Span)` -Creates an OpenTelemetry context from a Braintrust span export string. +Creates an OpenTelemetry context from a Braintrust span using W3C Trace Context. -**When to use it**: When Service A uses Braintrust and sends a span export to Service B that uses OpenTelemetry. This allows Service B to create spans as children of Service A's span. +**When to use it**: When Braintrust and OpenTelemetry code run in the same process and an OpenTelemetry operation should be a child of a Braintrust span. Across services, propagate the headers returned by `span.inject()` instead. ```typescript -import { contextFromSpanExport } from "@braintrust/otel"; +import { contextFromSpan } from "@braintrust/otel"; import { context } from "@opentelemetry/api"; -// Service A (Braintrust) exports span -const exportedSpan = await spanA.export(); - -// Service B (OpenTelemetry) imports context -const ctx = contextFromSpanExport(exportedSpan); +const ctx = contextFromSpan(spanA); await context.with(ctx, async () => { // OpenTelemetry spans created here will be children of Service A's span await tracer.startActiveSpan("service_b_operation", async (span) => { diff --git a/integrations/otel-js/src/context.ts b/integrations/otel-js/src/context.ts index 6db697c67..b61416e41 100644 --- a/integrations/otel-js/src/context.ts +++ b/integrations/otel-js/src/context.ts @@ -1,11 +1,10 @@ import { ContextManager, - BRAINTRUST_CURRENT_SPAN_STORE, - _internalIso as iso, type ContextParentSpanIds, type CurrentSpanStore, type Span, } from "braintrust"; +import { AsyncLocalStorage } from "node:async_hooks"; import { trace as otelTrace, context as otelContext } from "@opentelemetry/api"; import { getOtelParentFromSpan } from "./otel"; @@ -91,29 +90,11 @@ export class OtelContextManager extends ContextManager { return (otelContext as any)._getContextManager?.()._asyncLocalStorage; } - constructor() { - super(); - // Expose whichever ALS is in use via BRAINTRUST_CURRENT_SPAN_STORE so that - // TracingChannel's bindStore can propagate span context. We prefer OTEL's own - // ALS (AsyncLocalStorageContextManager._asyncLocalStorage) so that spans - // stored by runStores are visible to OTEL's context APIs. If the active OTEL - // context manager doesn't expose an ALS (e.g. AsyncHooksContextManager), we - // fall back to our own IsoAsyncLocalStorage and behave like the default - // BraintrustContextManager for TracingChannel binding. - // - // A lazy getter is required because the global OTEL context manager may not be - // registered until after this instance is constructed. - const self = this; - Object.defineProperty(this, BRAINTRUST_CURRENT_SPAN_STORE, { - get(): CurrentSpanStore { - const otelAls = self._getOtelAls(); - if (otelAls) return otelAls; - if (!self._ownAls) self._ownAls = iso.newAsyncLocalStorage(); - return self._ownAls; - }, - configurable: true, - enumerable: false, - }); + getCurrentSpanStore(): CurrentSpanStore { + const otelAls = this._getOtelAls(); + if (otelAls) return otelAls; + if (!this._ownAls) this._ownAls = new AsyncLocalStorage(); + return this._ownAls; } wrapSpanForStore(span: Span): unknown { diff --git a/integrations/otel-js/src/exporter.test.ts b/integrations/otel-js/src/exporter.test.ts index 94168963a..b0391222d 100644 --- a/integrations/otel-js/src/exporter.test.ts +++ b/integrations/otel-js/src/exporter.test.ts @@ -22,7 +22,7 @@ import { } from "@opentelemetry/sdk-trace-base"; import { BraintrustSpanProcessor } from "./otel"; import { createTracerProvider } from "../tests/utils"; -import { _exportsForTestingOnly } from "braintrust"; +import { _exportsForTestingOnly } from "../../../js/src/logger"; type OtlpTraceRequest = { resourceSpans?: Array<{ diff --git a/integrations/otel-js/src/id-gen.test.ts b/integrations/otel-js/src/id-gen.test.ts index b38e0afbc..19fcb6f93 100644 --- a/integrations/otel-js/src/id-gen.test.ts +++ b/integrations/otel-js/src/id-gen.test.ts @@ -1,6 +1,9 @@ import { expect, test, describe, beforeEach, afterEach } from "vitest"; -import { UUIDGenerator, getIdGenerator } from "braintrust"; -import { OTELIDGenerator } from "./otel"; +import { + OTELIDGenerator, + UUIDGenerator, + getIdGenerator, +} from "../../../js/src/id-gen"; import { setupOtelCompat, resetOtelCompat } from "."; describe("ID Generation", () => { diff --git a/integrations/otel-js/src/index.ts b/integrations/otel-js/src/index.ts index 9555cd32c..41f2b20cb 100644 --- a/integrations/otel-js/src/index.ts +++ b/integrations/otel-js/src/index.ts @@ -1,48 +1,20 @@ import { OtelContextManager } from "./context"; -import { _internalGetGlobalState } from "braintrust"; +import { configureContextManager } from "braintrust"; export { - contextFromSpanExport, + contextFromSpan, addSpanParentToBaggage, addParentToBaggage, parentFromHeaders, isRootSpan, } from "./otel"; -import { OTELIDGenerator } from "./otel"; -import { SpanComponentsV4 } from "braintrust/util"; - export { BraintrustSpanProcessor, BraintrustExporter } from "./otel"; -function resetBraintrustCompatCaches() { - const state = _internalGetGlobalState(); - if (!state) { - return; - } - - // Node/browser package initialization can cache native context and ID state - // before setupOtelCompat() runs. Reset both so subsequent lookups honor the - // compat-mode globals we are about to install. - (state as unknown as { _contextManager: unknown })._contextManager = null; - (state as unknown as { _idGenerator: unknown })._idGenerator = null; -} - export const setupOtelCompat = () => { - // eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/consistent-type-assertions - (globalThis as any).BRAINTRUST_CONTEXT_MANAGER = OtelContextManager; - // eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/consistent-type-assertions - (globalThis as any).BRAINTRUST_ID_GENERATOR = OTELIDGenerator; - // eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/consistent-type-assertions - (globalThis as any).BRAINTRUST_SPAN_COMPONENT = SpanComponentsV4; - resetBraintrustCompatCaches(); + configureContextManager(OtelContextManager); }; export const resetOtelCompat = () => { - // eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/consistent-type-assertions - (globalThis as any).BRAINTRUST_CONTEXT_MANAGER = undefined; - // eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/consistent-type-assertions - (globalThis as any).BRAINTRUST_ID_GENERATOR = undefined; - // eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/consistent-type-assertions - (globalThis as any).BRAINTRUST_SPAN_COMPONENT = undefined; - resetBraintrustCompatCaches(); + configureContextManager(undefined); }; diff --git a/integrations/otel-js/src/otel-compat.test.ts b/integrations/otel-js/src/otel-compat.test.ts index d06d9f843..35db904c6 100644 --- a/integrations/otel-js/src/otel-compat.test.ts +++ b/integrations/otel-js/src/otel-compat.test.ts @@ -6,14 +6,13 @@ */ import { beforeEach, afterEach, describe, expect, test } from "vitest"; +import { initLogger, currentSpan, Eval } from "braintrust"; import { - initLogger, - currentSpan, - getContextManager, - BRAINTRUST_CURRENT_SPAN_STORE, _exportsForTestingOnly, - Eval, -} from "braintrust"; + _internalStartSpan, + _internalWithParent, + getContextManager, +} from "../../../js/src/logger"; import { BasicTracerProvider, InMemorySpanExporter, @@ -264,15 +263,13 @@ describe("OTEL compatibility mode", () => { expect(cm.getCurrentSpan).toBeDefined(); }); - test("uses OtelContextManager when OTEL enabled", async () => { + test("uses OtelContextManager when OTEL enabled", () => { // Test that when OTEL is available and env var is set, we get OtelContextManager // Note: In test environment, OTEL packages may not be available even though // we checked OTEL_AVAILABLE. If the require fails in getContextManager, // it falls back to BraintrustContextManager, which is correct behavior. - // Clear module cache and re-import to get fresh context manager - const loggerModule = await import("braintrust?t=" + Date.now()); - const cm = loggerModule.getContextManager(); + const cm = getContextManager(); // If OTEL is truly available, we should get OtelContextManager // Otherwise, fallback to BraintrustContextManager is acceptable @@ -652,7 +649,7 @@ describe("OTEL compatibility mode", () => { parentSpan.end(); - const childSpan = logger.startSpan({ + const childSpan = _internalStartSpan({ name: "child-span-v4", parent: exported, }); @@ -678,11 +675,9 @@ describe("OTEL compatibility mode", () => { parentSpan.end(); // Use withParent helper with logger.startSpan to properly handle the exported parent - const { withParent } = await import("braintrust"); - - withParent(exported, () => { + _internalWithParent(exported, () => { // Pass parent explicitly in addition to withParent context - const childSpan = logger.startSpan({ + const childSpan = _internalStartSpan({ name: "child-span-v4-with-parent", parent: exported, }); @@ -720,12 +715,11 @@ describe("OtelContextManager TracingChannel integration", () => { resetOtelCompat(); }); - test("OtelContextManager exposes OTEL ALS via BRAINTRUST_CURRENT_SPAN_STORE", () => { + test("OtelContextManager exposes the OTEL ALS", () => { const cm = getContextManager(); expect(cm.constructor.name).toBe("OtelContextManager"); - // eslint-disable-next-line @typescript-eslint/no-explicit-any - const store = (cm as any)[BRAINTRUST_CURRENT_SPAN_STORE]; + const store = cm.getCurrentSpanStore()!; expect(store).toBeDefined(); expect(typeof store.run).toBe("function"); expect(typeof store.getStore).toBe("function"); @@ -756,8 +750,7 @@ describe("OtelContextManager TracingChannel integration", () => { test("store.run() with wrapSpanForStore output propagates span to currentSpan()", () => { const cm = getContextManager(); - // eslint-disable-next-line @typescript-eslint/no-explicit-any - const store = (cm as any)[BRAINTRUST_CURRENT_SPAN_STORE]; + const store = cm.getCurrentSpanStore()!; expect(store).toBeDefined(); const mockSpan = { @@ -778,8 +771,7 @@ describe("OtelContextManager TracingChannel integration", () => { test("nested store.run() calls maintain correct span chain", () => { const cm = getContextManager(); - // eslint-disable-next-line @typescript-eslint/no-explicit-any - const store = (cm as any)[BRAINTRUST_CURRENT_SPAN_STORE]; + const store = cm.getCurrentSpanStore()!; const parentSpan = { spanId: "1111111111111111", @@ -863,10 +855,9 @@ describe("OtelContextManager fallback ALS (AsyncHooksContextManager)", () => { resetOtelCompat(); }); - test("exposes fallback ALS via BRAINTRUST_CURRENT_SPAN_STORE when OTEL ALS is unavailable", () => { + test("exposes fallback ALS when OTEL ALS is unavailable", () => { const cm = getContextManager(); - // eslint-disable-next-line @typescript-eslint/no-explicit-any - const store = (cm as any)[BRAINTRUST_CURRENT_SPAN_STORE]; + const store = cm.getCurrentSpanStore()!; expect(store).toBeDefined(); expect(typeof store.run).toBe("function"); expect(typeof store.getStore).toBe("function"); @@ -889,8 +880,7 @@ describe("OtelContextManager fallback ALS (AsyncHooksContextManager)", () => { test("store.run() with span in fallback mode makes getCurrentSpan() work", () => { const cm = getContextManager(); - // eslint-disable-next-line @typescript-eslint/no-explicit-any - const store = (cm as any)[BRAINTRUST_CURRENT_SPAN_STORE]; + const store = cm.getCurrentSpanStore()!; const mockSpan = { spanId: "fedcba0987654321", @@ -938,29 +928,16 @@ describe("Distributed Tracing (BT → OTEL)", () => { resetOtelCompat(); }); - test("otelContextFromSpanExport parses BT span and creates OTEL context", async () => { - const { contextFromSpanExport: otelContextFromSpanExport } = - await import("./"); - const { SpanComponentsV4 } = await import("braintrust/util"); - const { SpanObjectTypeV3 } = await import("braintrust/util"); - - // Create a sample span export string + test("contextFromSpan creates an OTEL context", async () => { + const { contextFromSpan } = await import("./"); const rootSpanId = "a1b2c3d4e5f6789012345678abcdef01"; // 32 hex chars (16 bytes) const spanId = "a1b2c3d4e5f67890"; // 16 hex chars (8 bytes) - const objectId = "proj-123"; - - const components = new SpanComponentsV4({ - object_type: SpanObjectTypeV3.PROJECT_LOGS, - object_id: objectId, - row_id: "row-123", - span_id: spanId, - root_span_id: rootSpanId, - propagated_event: undefined, - }); - - const exportStr = components.toStr(); - - const ctx = otelContextFromSpanExport(exportStr); + const ctx = contextFromSpan({ + inject: () => ({ + traceparent: `00-${rootSpanId}-${spanId}-01`, + baggage: "braintrust.parent=project_id%3Aproj-123", + }), + } as any); // Verify that a valid context was created expect(ctx).toBeDefined(); @@ -985,8 +962,7 @@ describe("Distributed Tracing (BT → OTEL)", () => { test("BT span in Service A can be parent of OTEL span in Service B", async () => { const { tracer, exporter, processor } = setupOtelFixture("service-a-project"); - const { contextFromSpanExport: otelContextFromSpanExport } = - await import("./"); + const { contextFromSpan } = await import("./"); const projectName = "service-a-project"; const logger = initLogger({ projectName }); @@ -994,25 +970,23 @@ describe("Distributed Tracing (BT → OTEL)", () => { // ===== Service A: Create BT span and export ===== let serviceATraceId: string | undefined; let serviceASpanId: string | undefined; - let exportedContext: string | undefined; + let sourceSpan: import("braintrust").Span | undefined; await logger.traced( async (serviceASpan) => { serviceATraceId = serviceASpan.rootSpanId; serviceASpanId = serviceASpan.spanId; - - // Export context for sending to Service B (e.g., via HTTP header) - exportedContext = await serviceASpan.export(); + sourceSpan = serviceASpan; }, { name: "service_a_span" }, ); - expect(exportedContext).toBeDefined(); + expect(sourceSpan).toBeDefined(); expect(serviceATraceId).toBeDefined(); expect(serviceASpanId).toBeDefined(); // ===== Service B: Import context and create OTEL child span ===== - const ctx = otelContextFromSpanExport(exportedContext!); + const ctx = contextFromSpan(sourceSpan!); // Use context.with() to run code in the imported context await context.with(ctx, async () => { diff --git a/integrations/otel-js/src/otel.test.ts b/integrations/otel-js/src/otel.test.ts index 7522450da..e34e2a36b 100644 --- a/integrations/otel-js/src/otel.test.ts +++ b/integrations/otel-js/src/otel.test.ts @@ -26,13 +26,14 @@ import { parentFromHeaders, isRootSpan, } from "./otel"; -import { _exportsForTestingOnly, initLogger } from "braintrust"; +import { initLogger } from "braintrust"; +import { SpanComponentsV3, SpanComponentsV4 } from "../../../js/util"; +import { _exportsForTestingOnly } from "../../../js/src/logger"; import { base64ToUint8Array, getExportVersion, createTracerProvider, } from "../tests/utils"; -import { SpanComponentsV3, SpanComponentsV4 } from "braintrust/util"; import { setupOtelCompat, resetOtelCompat } from "."; const packageJson = JSON.parse( @@ -61,6 +62,13 @@ async function withEmptyBraintrustEnvFile( return withBraintrustEnvFile("", fn); } +function restoreEnvironment(snapshot: NodeJS.ProcessEnv): void { + for (const name of Object.keys(process.env)) { + if (!(name in snapshot)) delete process.env[name]; + } + Object.assign(process.env, snapshot); +} + describe("AISpanProcessor", () => { let memoryExporter: InMemorySpanExporter; let provider: BasicTracerProvider; @@ -605,7 +613,7 @@ describe("BraintrustSpanProcessor", () => { }); afterEach(() => { - process.env = originalEnv; + restoreEnvironment(originalEnv); vi.restoreAllMocks(); _exportsForTestingOnly.clearTestBackgroundLogger(); _exportsForTestingOnly.simulateLogoutForTests(); @@ -1008,7 +1016,7 @@ describe("BraintrustExporter", () => { }); afterEach(() => { - process.env = originalEnv; + restoreEnvironment(originalEnv); vi.restoreAllMocks(); _exportsForTestingOnly.clearTestBackgroundLogger(); _exportsForTestingOnly.simulateLogoutForTests(); @@ -1392,180 +1400,35 @@ describe("otel namespace helpers", () => { }); describe("parentFromHeaders", () => { - describe("valid inputs", () => { - it("should extract parent from headers with valid traceparent and braintrust.parent baggage", () => { - const headers = { - traceparent: - "00-12345678901234567890123456789012-1234567890123456-01", - baggage: "braintrust.parent=project_name:test", - }; - - const parent = parentFromHeaders(headers); - expect(parent).toBeDefined(); - // Parent string is base64-encoded SpanComponentsV4 - expect(typeof parent).toBe("string"); - expect(parent!.length).toBeGreaterThan(0); - }); - - it("should extract parent with project_id", () => { - const headers = { - traceparent: - "00-abcdef12345678901234567890123456-fedcba9876543210-01", - baggage: "braintrust.parent=project_id:my-project-id", - }; - - const parent = parentFromHeaders(headers); - expect(parent).toBeDefined(); - expect(typeof parent).toBe("string"); - expect(parent!.length).toBeGreaterThan(0); - }); - - it("should extract parent with experiment_id", () => { - const headers = { - traceparent: - "00-11111111111111111111111111111111-2222222222222222-01", - baggage: "braintrust.parent=experiment_id:my-experiment-id", - }; - - const parent = parentFromHeaders(headers); - expect(parent).toBeDefined(); - expect(typeof parent).toBe("string"); - expect(parent!.length).toBeGreaterThan(0); - }); + it("returns the opaque W3C context", () => { + const headers = { + traceparent: "00-12345678901234567890123456789012-1234567890123456-01", + tracestate: "vendor=value", + baggage: "braintrust.parent=project_name%3Atest", + }; + expect(parentFromHeaders(headers)).toEqual(headers); }); - describe("invalid inputs", () => { - it("should return undefined when traceparent is missing", () => { - const consoleSpy = vi - .spyOn(console, "error") - .mockImplementation(() => {}); - const headers = { - baggage: "braintrust.parent=project_name:test", - }; - - const parent = parentFromHeaders(headers); - expect(parent).toBeUndefined(); - expect(consoleSpy).toHaveBeenCalledWith( - "parentFromHeaders: No valid span context in headers", - ); - - consoleSpy.mockRestore(); - }); - - it("should return undefined when baggage is missing", () => { - const consoleSpy = vi - .spyOn(console, "warn") - .mockImplementation(() => {}); - const headers = { - traceparent: - "00-12345678901234567890123456789012-1234567890123456-01", - }; - - const parent = parentFromHeaders(headers); - expect(parent).toBeUndefined(); - expect(consoleSpy).toHaveBeenCalled(); - expect(consoleSpy.mock.calls[0][0]).toContain( - "braintrust.parent not found", - ); - - consoleSpy.mockRestore(); - }); - - it("should return undefined when braintrust.parent is missing from baggage", () => { - const consoleSpy = vi - .spyOn(console, "warn") - .mockImplementation(() => {}); - const headers = { - traceparent: - "00-12345678901234567890123456789012-1234567890123456-01", - baggage: "foo=bar,baz=qux", - }; - - const parent = parentFromHeaders(headers); - expect(parent).toBeUndefined(); - expect(consoleSpy).toHaveBeenCalled(); - expect(consoleSpy.mock.calls[0][0]).toContain( - "braintrust.parent not found", - ); - - consoleSpy.mockRestore(); - }); - - it("should return undefined when traceparent format is invalid", () => { - const consoleSpy = vi - .spyOn(console, "error") - .mockImplementation(() => {}); - const headers = { - traceparent: "invalid-traceparent", - baggage: "braintrust.parent=project_name:test", - }; - - const parent = parentFromHeaders(headers); - expect(parent).toBeUndefined(); - expect(consoleSpy).toHaveBeenCalledWith( - "parentFromHeaders: No valid span context in headers", - ); - - consoleSpy.mockRestore(); - }); - - it("should return undefined when trace_id is all zeros", () => { - const consoleSpy = vi - .spyOn(console, "error") - .mockImplementation(() => {}); - const headers = { - traceparent: - "00-00000000000000000000000000000000-1234567890123456-01", - baggage: "braintrust.parent=project_name:test", - }; - - const parent = parentFromHeaders(headers); - expect(parent).toBeUndefined(); - // OTEL's extract() validates and rejects invalid trace_id - expect(consoleSpy).toHaveBeenCalledWith( - "parentFromHeaders: No valid span context in headers", - ); - - consoleSpy.mockRestore(); - }); - - it("should return undefined when span_id is all zeros", () => { - const consoleSpy = vi - .spyOn(console, "error") - .mockImplementation(() => {}); - const headers = { - traceparent: - "00-12345678901234567890123456789012-0000000000000000-01", - baggage: "braintrust.parent=project_name:test", - }; - - const parent = parentFromHeaders(headers); - expect(parent).toBeUndefined(); - // OTEL's extract() validates and rejects invalid span_id - expect(consoleSpy).toHaveBeenCalledWith( - "parentFromHeaders: No valid span context in headers", - ); - - consoleSpy.mockRestore(); - }); - - it("should return undefined when braintrust.parent format is invalid", () => { - const consoleSpy = vi - .spyOn(console, "error") - .mockImplementation(() => {}); - const headers = { - traceparent: - "00-12345678901234567890123456789012-1234567890123456-01", - baggage: "braintrust.parent=invalid", - }; - - const parent = parentFromHeaders(headers); - expect(parent).toBeUndefined(); - // Should reach our validation if span context is valid, otherwise OTEL rejects it - expect(consoleSpy).toHaveBeenCalled(); + it("keeps a valid trace context without Braintrust baggage", () => { + const headers = { + traceparent: "00-12345678901234567890123456789012-1234567890123456-01", + }; + expect(parentFromHeaders(headers)).toEqual(headers); + }); - consoleSpy.mockRestore(); - }); + it.each([ + {}, + { traceparent: "invalid-traceparent" }, + { + traceparent: "00-00000000000000000000000000000000-1234567890123456-01", + }, + { + traceparent: "00-12345678901234567890123456789012-0000000000000000-01", + }, + ])("returns undefined for an invalid traceparent", (headers) => { + expect( + parentFromHeaders(headers as Record), + ).toBeUndefined(); }); }); }); @@ -1735,7 +1598,9 @@ describe("Otel Compat tests Integration", () => { expect(uuidSpan.spanId).toMatch(uuidRegex); uuidSpan.end(); - // Switch to OTEL compat (hex, wins over legacy). + // Switch to the default hex ID mode. OTEL compat only changes context + // management now; it does not override the core ID generator. + delete process.env.BRAINTRUST_LEGACY_IDS; setupOtelCompat(); _exportsForTestingOnly.resetIdGenStateForTests(); @@ -1941,26 +1806,6 @@ describe("export() format selection based on if otel is initialized", () => { expect(v4Parsed.data.object_type).toBeDefined(); }); - test("exported V4 span can be used as parent", async () => { - const testLogger = initLogger({ - projectName: "test-v4-parent", - apiKey: "test-key", - }); - - const parentSpan = testLogger.startSpan({ name: "parent-span-v4" }); - const exported = await parentSpan.export(); - parentSpan.end(); - - // Should be able to use V4 exported string as parent - const childSpan = testLogger.startSpan({ - name: "child-span-v4", - parent: exported, - }); - - expect(childSpan.rootSpanId).toBe(parentSpan.rootSpanId); - childSpan.end(); - }); - test("V4 format uses hex IDs (not UUIDs) when otel is initialized", async () => { _exportsForTestingOnly.resetIdGenStateForTests(); diff --git a/integrations/otel-js/src/otel.ts b/integrations/otel-js/src/otel.ts index dc7e23aca..bcc817017 100644 --- a/integrations/otel-js/src/otel.ts +++ b/integrations/otel-js/src/otel.ts @@ -1,14 +1,19 @@ -import { SpanComponentsV4, SpanObjectTypeV3 } from "braintrust/util"; +import { registerOtelFlush } from "braintrust/instrumentation"; import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-http"; +import { + CompositePropagator, + W3CBaggagePropagator, + W3CTraceContextPropagator, +} from "@opentelemetry/core"; import { context, Context, diag, trace, - TraceFlags, propagation, Span, + type TextMapGetter, } from "@opentelemetry/api"; import { SpanProcessor, @@ -18,10 +23,9 @@ import { type SpanExporter, } from "@opentelemetry/sdk-trace-base"; import { - IDGenerator, - _internalIso, currentSpan, - registerOtelFlush, + extractTraceContextFromHeaders, + type PropagationContext, type Span as BraintrustSpan, } from "braintrust"; @@ -45,6 +49,14 @@ const SYSTEM_ATTRIBUTE_NAMES = new Set([ "braintrust.context_json", ]); +const braintrustW3CPropagator = new CompositePropagator({ + propagators: [new W3CTraceContextPropagator(), new W3CBaggagePropagator()], +}); +const traceContextGetter: TextMapGetter> = { + keys: Object.keys, + get: (carrier, key) => carrier[key], +}; + /** * Custom filter function type for span filtering. * @param span - The span to evaluate @@ -277,45 +289,100 @@ function withSpanOriginAttributes( }); } +const BRAINTRUST_ENV_SEARCH_PARENT_LIMIT = 64; + +function getEnv(name: string): string | undefined { + if (typeof process === "undefined") return undefined; + const value = process.env[name]; + if (value?.trim()) return value; + if ( + name !== "BRAINTRUST_API_KEY" && + name !== "BRAINTRUST_ENVIRONMENT_TYPE" && + name !== "BRAINTRUST_ENVIRONMENT_NAME" + ) { + return value; + } + + if (typeof process.loadEnvFile !== "function") return undefined; + + let dir: string; + try { + dir = process.cwd(); + } catch { + return undefined; + } + + for (let depth = 0; depth <= BRAINTRUST_ENV_SEARCH_PARENT_LIMIT; depth++) { + try { + const separator = dir.includes("\\") ? "\\" : "/"; + process.loadEnvFile( + `${dir}${dir.endsWith(separator) ? "" : separator}.env.braintrust`, + ); + const fileValue = process.env[name]; + return fileValue?.trim() ? fileValue : undefined; + } catch (error) { + if ( + typeof error !== "object" || + error === null || + !("code" in error) || + error.code !== "ENOENT" + ) { + return undefined; + } + } + + const trimmed = dir.replace(/[\\/]+$/, ""); + const separatorIndex = Math.max( + trimmed.lastIndexOf("/"), + trimmed.lastIndexOf("\\"), + ); + const parent = + separatorIndex < 0 + ? dir + : separatorIndex === 0 + ? trimmed.slice(0, 1) + : trimmed.slice(0, separatorIndex); + if (parent === dir) break; + dir = parent; + } + return undefined; +} + function detectEnvironment(explicit?: { type?: string; name?: string; }): { type?: string; name?: string } | undefined { if (explicit) return explicit; - const envType = _internalIso.getEnv("BRAINTRUST_ENVIRONMENT_TYPE"); - const envName = _internalIso.getEnv("BRAINTRUST_ENVIRONMENT_NAME"); + const envType = getEnv("BRAINTRUST_ENVIRONMENT_TYPE"); + const envName = getEnv("BRAINTRUST_ENVIRONMENT_NAME"); if (envType || envName) { return { ...(envType ? { type: envType } : {}), ...(envName ? { name: envName } : {}), }; } - if (_internalIso.getEnv("GITHUB_ACTIONS")) - return { type: "ci", name: "github_actions" }; - if (_internalIso.getEnv("GITLAB_CI")) - return { type: "ci", name: "gitlab_ci" }; - if (_internalIso.getEnv("CIRCLECI")) return { type: "ci", name: "circleci" }; - if (_internalIso.getEnv("BUILDKITE")) - return { type: "ci", name: "buildkite" }; - if (_internalIso.getEnv("CI")) return { type: "ci", name: "ci" }; - if (_internalIso.getEnv("VERCEL")) return { type: "server", name: "vercel" }; - if (_internalIso.getEnv("NETLIFY")) - return { type: "server", name: "netlify" }; - const awsExecutionEnv = _internalIso.getEnv("AWS_EXECUTION_ENV"); + if (getEnv("GITHUB_ACTIONS")) return { type: "ci", name: "github_actions" }; + if (getEnv("GITLAB_CI")) return { type: "ci", name: "gitlab_ci" }; + if (getEnv("CIRCLECI")) return { type: "ci", name: "circleci" }; + if (getEnv("BUILDKITE")) return { type: "ci", name: "buildkite" }; + if (getEnv("CI")) return { type: "ci", name: "ci" }; + if (getEnv("VERCEL")) return { type: "server", name: "vercel" }; + if (getEnv("NETLIFY")) return { type: "server", name: "netlify" }; + const awsExecutionEnv = getEnv("AWS_EXECUTION_ENV"); if ( - _internalIso.getEnv("ECS_CONTAINER_METADATA_URI") || - _internalIso.getEnv("ECS_CONTAINER_METADATA_URI_V4") || + getEnv("ECS_CONTAINER_METADATA_URI") || + getEnv("ECS_CONTAINER_METADATA_URI_V4") || awsExecutionEnv?.startsWith("AWS_ECS_") ) { return { type: "server", name: "ecs" }; } if ( - _internalIso.getEnv("AWS_LAMBDA_FUNCTION_NAME") || + getEnv("AWS_LAMBDA_FUNCTION_NAME") || awsExecutionEnv?.startsWith("AWS_Lambda_") ) { return { type: "server", name: "aws_lambda" }; } - const nodeEnv = _internalIso.getEnv("NODE_ENV"); + const nodeEnv = getEnv("NODE_ENV"); if (!nodeEnv) return undefined; const normalizedNodeEnv = nodeEnv.toLowerCase(); if (normalizedNodeEnv === "production" || normalizedNodeEnv === "staging") { @@ -438,7 +505,7 @@ class LazyBraintrustOTLPTraceExporter implements SpanExporter { const apiKey = this.apiKey !== undefined ? this.apiKey - : await _internalIso.getBraintrustApiKey(); + : getEnv("BRAINTRUST_API_KEY"); if (!apiKey?.trim()) { throw new Error( "Braintrust API key is required. Set BRAINTRUST_API_KEY, define it in .env.braintrust, or pass apiKey option.", @@ -529,7 +596,7 @@ export class BraintrustSpanProcessor implements SpanProcessor { const apiKey = options.apiKey !== undefined ? options.apiKey - : _internalIso.getEnv("BRAINTRUST_API_KEY"); + : getEnv("BRAINTRUST_API_KEY"); // Get API URL from options or environment let apiUrl = @@ -668,132 +735,13 @@ export class BraintrustSpanProcessor implements SpanProcessor { } } -/** - * Create an OTEL context from a Braintrust span export string. - * - * Used for distributed tracing scenarios where a Braintrust span in one service - * needs to be the parent of an OTEL span in another service. - * - * @param exportStr - The string returned from span.export() - * @returns OTEL context that can be used when creating child spans - * - * @example - * ```typescript - * // Service A: Create BT span and export - * const span = logger.startSpan({ name: "service-a" }); - * const exportStr = await span.export(); - * // Send exportStr to Service B (e.g., via HTTP header) - * - * // Service B: Import context and create OTEL child - * import * as api from '@opentelemetry/api'; - * const ctx = contextFromSpanExport(exportStr); - * await api.context.with(ctx, async () => { - * await tracer.startActiveSpan("service-b", async (span) => { - * // This span is now a child of the Service A span - * span.end(); - * }); - * }); - * ``` - */ -export function contextFromSpanExport(exportStr: string): unknown { - // Parse the export string - const components = SpanComponentsV4.fromStr(exportStr); - - // Get trace and span IDs (already in hex format) - const traceIdHex = components.data.root_span_id; // 32 hex chars - const spanIdHex = components.data.span_id; // 16 hex chars - - if (!traceIdHex || !spanIdHex) { - throw new Error( - "Export string must contain root_span_id and span_id for distributed tracing", - ); - } - - // Create SpanContext marked as remote (critical for distributed tracing) - const spanContext = { - traceId: traceIdHex, - spanId: spanIdHex, - isRemote: true, - traceFlags: TraceFlags?.SAMPLED ?? 1, // SAMPLED flag - }; - - // Create NonRecordingSpan using wrapSpanContext and set in context - const nonRecordingSpan = trace.wrapSpanContext(spanContext); - let ctx = trace.setSpan(context.active(), nonRecordingSpan); - - // Construct braintrust.parent identifier - const braintrustParent = getBraintrustParent( - components.data.object_type, - components.data.object_id, - components.data.compute_object_metadata_args, +/** Create an OTEL context from a Braintrust span using W3C propagation. */ +export function contextFromSpan(span: BraintrustSpan): Context { + return braintrustW3CPropagator.extract( + context.active(), + span.inject(), + traceContextGetter, ); - - // Set braintrust.parent in baggage so it propagates automatically - if (braintrustParent) { - try { - // Try to set baggage if available - if (propagation) { - const baggage = - propagation.getBaggage(ctx) || propagation.createBaggage(); - ctx = propagation.setBaggage( - ctx, - baggage.setEntry("braintrust.parent", { - value: braintrustParent, - }), - ); - } - } catch (error) { - console.error( - "Failed to set braintrust.parent in baggage during context import:", - error, - ); - } - } - - return ctx; -} - -/** - * Construct a braintrust.parent identifier string from span components. - * - * @param objectType - Type of parent object (PROJECT_LOGS or EXPERIMENT) - * @param objectId - Resolved object ID (project_id or experiment_id) - * @param computeArgs - Optional dict with project_name/project_id for unresolved cases - * @returns String like "project_id:abc", "project_name:my-proj", "experiment_id:exp-123", or undefined - */ -function getBraintrustParent( - objectType: number, - objectId?: string | null, - computeArgs?: Record | null, -): string | undefined { - if (!objectType) { - return undefined; - } - - if (objectType === SpanObjectTypeV3.PROJECT_LOGS) { - if (objectId) { - return `project_id:${objectId}`; - } else if (computeArgs) { - const projectId = computeArgs["project_id"]; - const projectName = computeArgs["project_name"]; - if (typeof projectId === "string") { - return `project_id:${projectId}`; - } else if (typeof projectName === "string") { - return `project_name:${projectName}`; - } - } - } else if (objectType === SpanObjectTypeV3.EXPERIMENT) { - if (objectId) { - return `experiment_id:${objectId}`; - } else if (computeArgs) { - const experimentId = computeArgs["experiment_id"]; - if (typeof experimentId === "string") { - return `experiment_id:${experimentId}`; - } - } - } - - return undefined; } /** @@ -806,46 +754,16 @@ function getBraintrustParent( export function getOtelParentFromSpan( span: BraintrustSpan, ): string | undefined { - const parent = span.getParentInfo(); - - if (!parent || !parent.objectType || !parent.objectId) { - return undefined; - } - try { - if (parent.objectType === SpanObjectTypeV3.PROJECT_LOGS) { - const syncResult = parent.objectId.getSync(); - const id = syncResult?.value; - const args = parent.computeObjectMetadataArgs; - - if (id) { - return `project_id:${id}`; - } - - const projectName = args?.project_name; - if (typeof projectName === "string") { - return `project_name:${projectName}`; - } - } else if (parent.objectType === SpanObjectTypeV3.EXPERIMENT) { - const syncResult = parent.objectId.getSync(); - const id = syncResult?.value; - - // Debug details for experiment - // eslint-disable-next-line no-console - console.debug("[getOtelParentFromSpan] EXPERIMENT", { id }); - - if (id) { - // eslint-disable-next-line no-console - console.debug( - "[getOtelParentFromSpan] EXPERIMENT using experiment_id", - { id }, - ); - return `experiment_id:${id}`; + if (typeof span.inject !== "function") return undefined; + const baggage = span.inject().baggage; + for (const member of baggage?.split(",") ?? []) { + const [rawKey, rawValue] = member.trim().split("=", 2); + if (rawKey === "braintrust.parent" && rawValue) { + return decodeURIComponent(rawValue.split(";", 1)[0]); } } } catch (e) { - // Debug: unexpected error reading parent info - // eslint-disable-next-line no-console console.warn("[getOtelParentFromSpan] error extracting parent", e); } return undefined; @@ -1045,188 +963,9 @@ export function addSpanParentToBaggage( return addParentToBaggage(parentValue, ctx); } -/** - * Extract a Braintrust-compatible parent string from W3C Trace Context headers. - * - * This converts OTEL trace context headers (traceparent/baggage) into a format - * that can be passed as the 'parent' parameter to Braintrust's traced() method. - * - * @param headers - Dictionary with 'traceparent' and optionally 'baggage' keys - * @returns Braintrust V4 export string that can be used as parent parameter, - * or undefined if no valid span context is found or braintrust.parent is missing. - * - * @example - * ```typescript - * import { initLogger } from "braintrust"; - * import { parentFromHeaders } from "@braintrust/otel"; - * - * // Service C receives headers from Service B - * const headers = { traceparent: '00-trace_id-span_id-01', baggage: '...' }; - * const parent = parentFromHeaders(headers); - * - * const logger = initLogger({ projectName: "my-project" }); - * await logger.traced(async (span) => { - * span.log({ input: "BT span as child of OTEL parent" }); - * }, { name: "service_c", parent }); - * ``` - */ +/** Extract an opaque Braintrust parent context from W3C headers. */ export function parentFromHeaders( headers: Record, -): string | undefined { - try { - if (!propagation) { - console.error("OTEL propagation API not available"); - return undefined; - } - - // Extract context from headers using W3C Trace Context propagator - // This parses both traceparent and baggage headers - const ctx = propagation.extract(context.active(), headers); - - // Get span context directly from the extracted context - const spanContext = trace.getSpanContext(ctx); - if (!spanContext) { - console.error("parentFromHeaders: No valid span context in headers"); - return undefined; - } - - // Get trace_id and span_id from span context - // eslint-disable-next-line @typescript-eslint/consistent-type-assertions - const traceIdHex = spanContext.traceId as string; - // eslint-disable-next-line @typescript-eslint/consistent-type-assertions - const spanIdHex = spanContext.spanId as string; - - // Validate trace_id and span_id are not all zeros - if ( - !traceIdHex || - typeof traceIdHex !== "string" || - traceIdHex === "00000000000000000000000000000000" - ) { - console.error("parentFromHeaders: Invalid trace_id (all zeros)"); - return undefined; - } - if ( - !spanIdHex || - typeof spanIdHex !== "string" || - spanIdHex === "0000000000000000" - ) { - console.error("parentFromHeaders: Invalid span_id (all zeros)"); - return undefined; - } - - // Get braintrust.parent from baggage - const baggage = propagation.getBaggage(ctx); - const braintrustParent = baggage?.getEntry("braintrust.parent")?.value; - - if (!braintrustParent) { - console.warn( - "parentFromHeaders: braintrust.parent not found in OTEL baggage. " + - "Cannot create Braintrust parent without project information. " + - "Ensure the OTEL span sets braintrust.parent in baggage before exporting headers.", - ); - return undefined; - } - - // Parse braintrust.parent to extract object_type and object_id - let objectType: number | undefined; - let objectId: string | undefined; - let computeArgs: Record | undefined; - - // Parse braintrust.parent format: "project_id:abc", "project_name:xyz", or "experiment_id:123" - if (braintrustParent.startsWith("project_id:")) { - objectType = SpanObjectTypeV3.PROJECT_LOGS; - objectId = braintrustParent.substring("project_id:".length); - if (!objectId) { - console.error( - `parentFromHeaders: Invalid braintrust.parent format (empty project_id): ${braintrustParent}`, - ); - return undefined; - } - } else if (braintrustParent.startsWith("project_name:")) { - objectType = SpanObjectTypeV3.PROJECT_LOGS; - const projectName = braintrustParent.substring("project_name:".length); - if (!projectName) { - console.error( - `parentFromHeaders: Invalid braintrust.parent format (empty project_name): ${braintrustParent}`, - ); - return undefined; - } - computeArgs = { project_name: projectName }; - } else if (braintrustParent.startsWith("experiment_id:")) { - objectType = SpanObjectTypeV3.EXPERIMENT; - objectId = braintrustParent.substring("experiment_id:".length); - if (!objectId) { - console.error( - `parentFromHeaders: Invalid braintrust.parent format (empty experiment_id): ${braintrustParent}`, - ); - return undefined; - } - } else { - console.error( - `parentFromHeaders: Invalid braintrust.parent format: ${braintrustParent}. ` + - "Expected format: 'project_id:ID', 'project_name:NAME', or 'experiment_id:ID'", - ); - return undefined; - } - - // Create SpanComponentsV4 and export as string - const componentsData: { - object_type: number; - object_id?: string | null; - compute_object_metadata_args?: Record | null; - row_id: string; - span_id: string; - root_span_id: string; - } = { - object_type: objectType, - row_id: "otel", // Dummy row_id to enable span_id/root_span_id fields - span_id: spanIdHex, - root_span_id: traceIdHex, - }; - - // Add either object_id or compute_object_metadata_args, not both - if (computeArgs) { - componentsData.compute_object_metadata_args = computeArgs; - } else { - componentsData.object_id = objectId; - } - - // eslint-disable-next-line @typescript-eslint/consistent-type-assertions, @typescript-eslint/no-explicit-any - const components = new SpanComponentsV4(componentsData as any); - - return components.toStr(); - } catch (error) { - console.error("parentFromHeaders: Error parsing headers:", error); - return undefined; - } -} - -function generateHexId(bytes: number): string { - let result = ""; - for (let i = 0; i < bytes; i++) { - result += Math.floor(Math.random() * 256) - .toString(16) - .padStart(2, "0"); - } - return result; -} - -/** - * ID generator that generates OpenTelemetry-compatible IDs - * Uses hex strings for compatibility with OpenTelemetry systems - */ -export class OTELIDGenerator extends IDGenerator { - getSpanId(): string { - // Generate 8 random bytes and convert to hex (16 characters) - return generateHexId(8); - } - - getTraceId(): string { - // Generate 16 random bytes and convert to hex (32 characters) - return generateHexId(16); - } - - shareRootSpanId(): boolean { - return false; - } +): PropagationContext | undefined { + return extractTraceContextFromHeaders(headers); } diff --git a/integrations/otel-js/tests/utils.ts b/integrations/otel-js/tests/utils.ts index 95d30b3b9..a435f7515 100644 --- a/integrations/otel-js/tests/utils.ts +++ b/integrations/otel-js/tests/utils.ts @@ -139,7 +139,20 @@ export function logOtelVersions(version: "v1" | "v2" | "parent"): void { * which is critical for testing with different OpenTelemetry versions. */ export function createOtelAliases(cwd: string): Record { + const repositoryRoot = + detectOtelVersion(cwd) === "parent" + ? path.resolve(cwd, "../..") + : path.resolve(cwd, "../../.."); return { + "@braintrust/otel": path.resolve( + repositoryRoot, + "integrations/otel-js/src/index.ts", + ), + "braintrust/instrumentation": path.resolve( + repositoryRoot, + "js/src/instrumentation/index.ts", + ), + braintrust: path.resolve(repositoryRoot, "js/src/node/index.ts"), "@opentelemetry/api": path.resolve(cwd, "node_modules/@opentelemetry/api"), "@opentelemetry/core": path.resolve( cwd, diff --git a/integrations/otel-js/tsconfig.json b/integrations/otel-js/tsconfig.json index 8cfc6d9ae..352d6b9f7 100644 --- a/integrations/otel-js/tsconfig.json +++ b/integrations/otel-js/tsconfig.json @@ -2,9 +2,9 @@ "compilerOptions": { "declaration": true, "lib": ["es2022"], - "module": "commonjs", + "module": "esnext", "target": "es2022", - "moduleResolution": "node", + "moduleResolution": "bundler", "strict": true, "noUnusedLocals": true, "esModuleInterop": true, diff --git a/integrations/otel-js/tsconfig.test.json b/integrations/otel-js/tsconfig.test.json index eff2845e0..ad0f1bdc1 100644 --- a/integrations/otel-js/tsconfig.test.json +++ b/integrations/otel-js/tsconfig.test.json @@ -6,7 +6,9 @@ "types": ["node"], "baseUrl": ".", "paths": { - "@braintrust/otel": ["src/index.ts"] + "@braintrust/otel": ["src/index.ts"], + "braintrust": ["../../js/src/node/index.ts"], + "braintrust/instrumentation": ["../../js/src/instrumentation/index.ts"] } }, "include": ["src/**/*.test.ts", "tests/**/*.ts"], diff --git a/integrations/otel-js/vitest.config.ts b/integrations/otel-js/vitest.config.ts index 5da49856a..bf4fb3a75 100644 --- a/integrations/otel-js/vitest.config.ts +++ b/integrations/otel-js/vitest.config.ts @@ -1,5 +1,6 @@ import { defineConfig } from "vitest/config"; import { readFileSync } from "node:fs"; +import { createOtelAliases } from "./tests/utils"; const packageJson = JSON.parse( readFileSync(new URL("./package.json", import.meta.url), "utf8"), @@ -9,6 +10,7 @@ export default defineConfig({ define: { __BRAINTRUST_OTEL_VERSION__: JSON.stringify(packageJson.version), }, + resolve: { alias: createOtelAliases(process.cwd()) }, test: { reporters: ["default"], include: ["tests/**/*.test.ts", "src/**/*.test.ts"], diff --git a/integrations/temporal-js/src/interceptors.ts b/integrations/temporal-js/src/interceptors.ts index 70706872d..64d09dfc7 100644 --- a/integrations/temporal-js/src/interceptors.ts +++ b/integrations/temporal-js/src/interceptors.ts @@ -6,15 +6,9 @@ import type { ActivityInterceptors, } from "@temporalio/worker"; import type { WorkflowClientInterceptor } from "@temporalio/client"; -import { defaultPayloadConverter } from "@temporalio/common"; import * as braintrust from "braintrust"; -import { SpanComponentsV4 } from "braintrust/util"; -import { getWorkflowSpanExport } from "./sinks"; -import { - BRAINTRUST_SPAN_HEADER, - BRAINTRUST_WORKFLOW_SPAN_ID_HEADER, - deserializeHeaderValue, -} from "./utils"; +import { getWorkflowTraceContext } from "./sinks"; +import { deserializeTraceContext, serializeTraceContext } from "./utils"; /** * Create a client interceptor that propagates Braintrust span context to workflows. @@ -25,19 +19,13 @@ export function createBraintrustClientInterceptor(): WorkflowClientInterceptor { async start(input, next) { const span = braintrust.currentSpan(); if (span) { - const exported = await span.export(); - if (exported) { - const payload = defaultPayloadConverter.toPayload(exported); - if (payload) { - return next({ - ...input, - headers: { - ...input.headers, - [BRAINTRUST_SPAN_HEADER]: payload, - }, - }); - } - } + return next({ + ...input, + headers: { + ...input.headers, + ...serializeTraceContext(braintrust.injectTraceContext()), + }, + }); } return next(input); }, @@ -47,19 +35,13 @@ export function createBraintrustClientInterceptor(): WorkflowClientInterceptor { async signalWithStart(input, next) { const span = braintrust.currentSpan(); if (span) { - const exported = await span.export(); - if (exported) { - const payload = defaultPayloadConverter.toPayload(exported); - if (payload) { - return next({ - ...input, - headers: { - ...input.headers, - [BRAINTRUST_SPAN_HEADER]: payload, - }, - }); - } - } + return next({ + ...input, + headers: { + ...input.headers, + ...serializeTraceContext(braintrust.injectTraceContext()), + }, + }); } return next(input); }, @@ -79,76 +61,13 @@ class BraintrustActivityInterceptor implements ActivityInboundCallsInterceptor { const info = this.ctx.info; const runId = info.workflowExecution.runId; - // Try to get workflow span export - first check local Map, then headers - let parent: string | undefined; + // Prefer the live workflow span on the same worker, then fall back to the + // W3C context propagated through Temporal headers. + let parent: braintrust.PropagationContext | undefined; // Check if we have the workflow span export locally (same worker as workflow) - const spanExportPromise = getWorkflowSpanExport(runId); - if (spanExportPromise) { - try { - parent = await spanExportPromise; - } catch { - // Ignore errors, fall through to header check - } - } - - // For cross-worker activities: construct parent from workflow span ID + client context - if (!parent && input.headers) { - const workflowSpanId = deserializeHeaderValue( - input.headers[BRAINTRUST_WORKFLOW_SPAN_ID_HEADER], - ); - const clientContext = deserializeHeaderValue( - input.headers[BRAINTRUST_SPAN_HEADER], - ); - - if (workflowSpanId && clientContext) { - try { - const clientComponents = SpanComponentsV4.fromStr(clientContext); - const clientData = clientComponents.data; - - // We can only construct a workflow parent if we have: - // 1. Tracing context (root_span_id) - // 2. Object metadata (object_id or compute_object_metadata_args) - const hasTracingContext = !!clientData.root_span_id; - const hasObjectMetadata = - !!clientData.object_id || !!clientData.compute_object_metadata_args; - - if (hasTracingContext && hasObjectMetadata) { - // Construct workflow parent with the workflow's span ID - // IMPORTANT: row_id must match span_id for the parent span - // Must provide EITHER object_id OR compute_object_metadata_args, not both - const workflowParentBase = { - object_type: clientData.object_type, - propagated_event: clientData.propagated_event, - row_id: workflowSpanId, // Use workflow's row_id, not client's - span_id: workflowSpanId, // Use workflow's span_id, not client's - root_span_id: clientData.root_span_id, // Keep same trace - }; - const workflowComponents = clientData.object_id - ? new SpanComponentsV4({ - ...workflowParentBase, - object_id: clientData.object_id, - }) - : new SpanComponentsV4({ - ...workflowParentBase, - compute_object_metadata_args: - clientData.compute_object_metadata_args!, - }); - - parent = workflowComponents.toStr(); - } else { - // Client context doesn't have root_span_id, use it directly - parent = clientContext; - } - } catch { - // Fall back to client context if parsing fails - parent = clientContext; - } - } else if (clientContext) { - // No workflow span ID, use client context directly - parent = clientContext; - } - } + parent = + getWorkflowTraceContext(runId) ?? deserializeTraceContext(input.headers); const span = braintrust.startSpan({ name: `temporal.activity.${info.activityType}`, diff --git a/integrations/temporal-js/src/sinks.ts b/integrations/temporal-js/src/sinks.ts index bd2967ca4..ee8dc8891 100644 --- a/integrations/temporal-js/src/sinks.ts +++ b/integrations/temporal-js/src/sinks.ts @@ -6,24 +6,26 @@ import * as braintrust from "braintrust"; // NOTE: WorkflowInfo is NOT included here - it's automatically injected by the runtime export interface BraintrustSinks extends Sinks { braintrust: { - workflowStarted(parentContext?: string, workflowSpanId?: string): void; + workflowStarted( + parentContext?: braintrust.PropagationContext, + workflowSpanId?: string, + ): void; workflowCompleted(error?: string): void; }; } // Active workflow spans tracked by run ID const workflowSpans = new Map(); -// Workflow span exports tracked by run ID (as promises for async export) -const workflowSpanExports = new Map>(); +const workflowTraceContexts = new Map(); /** * Get the exported span context for a workflow by run ID. * Activities on the same worker can use this to parent to the workflow span. */ -export function getWorkflowSpanExport( +export function getWorkflowTraceContext( runId: string, -): Promise | undefined { - return workflowSpanExports.get(runId); +): braintrust.PropagationContext | undefined { + return workflowTraceContexts.get(runId); } /** @@ -36,7 +38,7 @@ export function createBraintrustSinks(): InjectedSinks { workflowStarted: { fn: ( info: WorkflowInfo, - parentContext?: string, + parentContext?: braintrust.PropagationContext, workflowSpanId?: string, ) => { const span = braintrust.startSpan({ @@ -53,7 +55,7 @@ export function createBraintrustSinks(): InjectedSinks { }, }); workflowSpans.set(info.runId, span); - workflowSpanExports.set(info.runId, span.export()); + workflowTraceContexts.set(info.runId, span.inject()); }, callDuringReplay: false, }, @@ -66,7 +68,7 @@ export function createBraintrustSinks(): InjectedSinks { } span.end(); workflowSpans.delete(info.runId); - workflowSpanExports.delete(info.runId); + workflowTraceContexts.delete(info.runId); } }, callDuringReplay: false, diff --git a/integrations/temporal-js/src/temporal.test.ts b/integrations/temporal-js/src/temporal.test.ts index 04b9c43c0..f8a8732f4 100644 --- a/integrations/temporal-js/src/temporal.test.ts +++ b/integrations/temporal-js/src/temporal.test.ts @@ -2,15 +2,15 @@ import { expect, test, describe } from "vitest"; import { serializeHeaderValue, deserializeHeaderValue, - BRAINTRUST_SPAN_HEADER, - BRAINTRUST_WORKFLOW_SPAN_HEADER, - BRAINTRUST_WORKFLOW_SPAN_ID_HEADER, + deserializeTraceContext, + serializeTraceContext, + withParentSpanId, } from "./utils"; import { SpanComponentsV3, SpanComponentsV4, SpanObjectTypeV3, -} from "braintrust/util"; +} from "../../../js/util"; import { BraintrustTemporalPlugin, createBraintrustTemporalPlugin, @@ -69,12 +69,29 @@ describe("temporal header utilities", () => { } }); - test("header constants are defined", () => { - expect(BRAINTRUST_SPAN_HEADER).toBe("_braintrust-span"); - expect(BRAINTRUST_WORKFLOW_SPAN_HEADER).toBe("_braintrust-workflow-span"); - expect(BRAINTRUST_WORKFLOW_SPAN_ID_HEADER).toBe( - "_braintrust-workflow-span-id", + test("round-trips W3C trace context through Temporal payloads", () => { + const context = { + traceparent: "00-12345678901234567890123456789012-1234567890123456-01", + tracestate: "vendor=value", + baggage: "braintrust.parent=project_name%3Atest", + }; + expect(deserializeTraceContext(serializeTraceContext(context))).toEqual( + context, + ); + }); + + test("reparents W3C context to the workflow span", () => { + const context = withParentSpanId( + { + traceparent: "00-12345678901234567890123456789012-1234567890123456-01", + baggage: "braintrust.parent=project_name%3Atest", + }, + "abcdefabcdefabcd", ); + expect(context).toEqual({ + traceparent: "00-12345678901234567890123456789012-abcdefabcdefabcd-01", + baggage: "braintrust.parent=project_name%3Atest", + }); }); }); diff --git a/integrations/temporal-js/src/utils.ts b/integrations/temporal-js/src/utils.ts index dc5add88a..51ffcae33 100644 --- a/integrations/temporal-js/src/utils.ts +++ b/integrations/temporal-js/src/utils.ts @@ -1,9 +1,7 @@ import type { Payload } from "@temporalio/common"; +import type { PropagationContext } from "braintrust"; -export const BRAINTRUST_SPAN_HEADER = "_braintrust-span"; -export const BRAINTRUST_WORKFLOW_SPAN_HEADER = "_braintrust-workflow-span"; -export const BRAINTRUST_WORKFLOW_SPAN_ID_HEADER = - "_braintrust-workflow-span-id"; +const TRACE_CONTEXT_HEADERS = ["traceparent", "tracestate", "baggage"] as const; export function serializeHeaderValue(value: string): Payload { return { @@ -27,3 +25,41 @@ export function deserializeHeaderValue( return undefined; } } + +export function serializeTraceContext( + context: PropagationContext, +): Record { + return Object.fromEntries( + TRACE_CONTEXT_HEADERS.flatMap((name) => { + const value = context[name]; + return value ? [[name, serializeHeaderValue(value)]] : []; + }), + ); +} + +export function deserializeTraceContext( + headers: Record | undefined, +): PropagationContext | undefined { + if (!headers) return undefined; + const context = Object.fromEntries( + TRACE_CONTEXT_HEADERS.flatMap((name) => { + const value = deserializeHeaderValue(headers[name]); + return value ? [[name, value]] : []; + }), + ); + return context.traceparent ? context : undefined; +} + +export function withParentSpanId( + context: PropagationContext, + spanId: string, +): PropagationContext | undefined { + const match = context.traceparent.match( + /^([0-9a-f]{2})-([0-9a-f]{32})-[0-9a-f]{16}-([0-9a-f]{2})$/, + ); + if (!match) return undefined; + return { + ...context, + traceparent: `${match[1]}-${match[2]}-${spanId}-${match[3]}`, + }; +} diff --git a/integrations/temporal-js/src/workflow-interceptors.ts b/integrations/temporal-js/src/workflow-interceptors.ts index 914dcc154..b7bc0a2b8 100644 --- a/integrations/temporal-js/src/workflow-interceptors.ts +++ b/integrations/temporal-js/src/workflow-interceptors.ts @@ -12,7 +12,6 @@ import { WorkflowExecuteInput, Next, proxySinks, - workflowInfo, uuid4, } from "@temporalio/workflow"; import type { @@ -21,13 +20,12 @@ import type { StartChildWorkflowExecutionInput, } from "@temporalio/workflow"; import type { Payload } from "@temporalio/common"; +import type { PropagationContext } from "braintrust"; import type { BraintrustSinks } from "./sinks"; import { - BRAINTRUST_SPAN_HEADER, - BRAINTRUST_WORKFLOW_SPAN_HEADER, - BRAINTRUST_WORKFLOW_SPAN_ID_HEADER, - serializeHeaderValue, - deserializeHeaderValue, + deserializeTraceContext, + serializeTraceContext, + withParentSpanId, } from "./utils"; const { braintrust } = proxySinks(); @@ -37,7 +35,7 @@ const { braintrust } = proxySinks(); * Created per-workflow by the factory function to avoid global state issues. */ interface WorkflowSpanState { - parentContext: string | undefined; + parentContext: PropagationContext | undefined; spanId: string | undefined; } @@ -49,15 +47,13 @@ class BraintrustWorkflowInboundInterceptor implements WorkflowInboundCallsInterc next: Next, ): Promise { // Extract parent context from headers - const parentContext = input.headers - ? deserializeHeaderValue(input.headers[BRAINTRUST_SPAN_HEADER]) - : undefined; + const parentContext = deserializeTraceContext(input.headers); // Store for the outbound interceptor to forward to activities this.state.parentContext = parentContext; // Generate a deterministic spanId for the workflow span - this.state.spanId = uuid4(); + this.state.spanId = uuid4().replace(/-/g, "").slice(0, 16); // Create workflow span via sink (only called if not replaying) // NOTE: WorkflowInfo is injected automatically by the runtime @@ -78,27 +74,11 @@ class BraintrustWorkflowOutboundInterceptor implements WorkflowOutboundCallsInte constructor(private state: WorkflowSpanState) {} private getHeaders(): Record { - const info = workflowInfo(); - const headers: Record = {}; - - // Pass runId so activity can look up workflow span on same worker - headers[BRAINTRUST_WORKFLOW_SPAN_HEADER] = serializeHeaderValue(info.runId); - - // Pass workflow span ID for cross-worker activities to construct parent - if (this.state.spanId) { - headers[BRAINTRUST_WORKFLOW_SPAN_ID_HEADER] = serializeHeaderValue( - this.state.spanId, - ); - } - - // Pass client context for cross-worker activities to construct parent - if (this.state.parentContext) { - headers[BRAINTRUST_SPAN_HEADER] = serializeHeaderValue( - this.state.parentContext, - ); - } - - return headers; + const context = + this.state.parentContext && this.state.spanId + ? withParentSpanId(this.state.parentContext, this.state.spanId) + : undefined; + return context ? serializeTraceContext(context) : {}; } scheduleActivity( diff --git a/integrations/temporal-js/tsconfig.json b/integrations/temporal-js/tsconfig.json index 8cfc6d9ae..352d6b9f7 100644 --- a/integrations/temporal-js/tsconfig.json +++ b/integrations/temporal-js/tsconfig.json @@ -2,9 +2,9 @@ "compilerOptions": { "declaration": true, "lib": ["es2022"], - "module": "commonjs", + "module": "esnext", "target": "es2022", - "moduleResolution": "node", + "moduleResolution": "bundler", "strict": true, "noUnusedLocals": true, "esModuleInterop": true, diff --git a/integrations/temporal-js/tsup.config.ts b/integrations/temporal-js/tsup.config.ts index e37c36c4b..6563ce288 100644 --- a/integrations/temporal-js/tsup.config.ts +++ b/integrations/temporal-js/tsup.config.ts @@ -7,7 +7,6 @@ export default defineConfig([ outDir: "dist", external: [ "braintrust", - "braintrust/util", "@braintrust/temporal/workflow-interceptors", "@temporalio/activity", "@temporalio/client", diff --git a/integrations/vercel-ai-sdk/src/adapter.test.ts b/integrations/vercel-ai-sdk/src/adapter.test.ts index 14d9e698e..99fcb5ee7 100644 --- a/integrations/vercel-ai-sdk/src/adapter.test.ts +++ b/integrations/vercel-ai-sdk/src/adapter.test.ts @@ -1,8 +1,15 @@ -import { BraintrustStreamChunk } from "braintrust"; +import { BraintrustStream } from "braintrust"; import { ReadableStream, WritableStream } from "stream/web"; import { expect, test } from "vitest"; import { BraintrustAdapter } from "."; +type BraintrustStreamChunk = + ReturnType extends ReadableStream< + infer Chunk + > + ? Chunk + : never; + test("text callbacks", async () => { const textStream = new ReadableStream({ start(controller) { diff --git a/integrations/vercel-ai-sdk/src/adapter.ts b/integrations/vercel-ai-sdk/src/adapter.ts index 7bb36ffa4..ee1c9d92b 100644 --- a/integrations/vercel-ai-sdk/src/adapter.ts +++ b/integrations/vercel-ai-sdk/src/adapter.ts @@ -1,4 +1,4 @@ -import { BraintrustStream, BraintrustStreamChunk } from "braintrust"; +import { BraintrustStream } from "braintrust"; export interface AIStreamCallbacksAndOptions { onStart?: () => void | Promise; @@ -15,6 +15,13 @@ function formatStreamPart(type: "text" | "data", value: unknown): string { import { ReadableStream, TransformStream } from "stream/web"; +type BraintrustStreamChunk = + ReturnType extends ReadableStream< + infer Chunk + > + ? Chunk + : never; + export type BraintrustStreamOrReadable = | BraintrustStream | ReadableStream @@ -47,16 +54,6 @@ export function toDataStreamResponse( }); } -/** - * @deprecated Use `toDataStreamResponse` instead. - */ -export function toAIStreamResponse( - stream: BraintrustStreamOrReadable, - init?: ResponseInit, -): Response { - return toDataStreamResponse(stream, init); -} - function btStreamToAISDKTransformStream( callbacks?: AIStreamCallbacksAndOptions, ) { diff --git a/js/README.md b/js/README.md index da6da81a6..a350c1423 100644 --- a/js/README.md +++ b/js/README.md @@ -136,7 +136,7 @@ The wrappers are composable and idempotent. They preserve LangSmith behavior, in Disable LangSmith instrumentation in code or through the environment: ```typescript -import { configureInstrumentation } from "braintrust"; +import { configureInstrumentation } from "braintrust/instrumentation"; configureInstrumentation({ integrations: { langsmith: false } }); ``` @@ -153,7 +153,7 @@ When Braintrust LangChain/LangGraph instrumentation is enabled, LangSmith runs s See the [Migrate from v2.x to v3.x guide](https://www.braintrust.dev/docs/reference/sdks/typescript/migrations/v2-to-v3). -In 3.x, browser usage should move to `@braintrust/browser` instead of relying on the legacy `braintrust/browser` path. +For direct browser imports, use `@braintrust/browser`. The package root still selects its browser build through export conditions, but `braintrust/browser` is no longer published. ### Upgrading from 1.x to 2.x diff --git a/js/package.json b/js/package.json index 2f9375376..c62e2ce92 100644 --- a/js/package.json +++ b/js/package.json @@ -46,11 +46,6 @@ "require": "./dist/edge-light.js", "default": "./dist/edge-light.mjs" }, - "./browser": { - "import": "./dist/browser.mjs", - "require": "./dist/browser.js", - "default": "./dist/browser.mjs" - }, "./apply-auto-instrumentation": { "types": "./dist/apply-auto-instrumentation.d.ts", "edge-light": "./dist/apply-auto-instrumentation.browser.mjs", @@ -76,12 +71,6 @@ "module": "./dist/index.mjs", "require": "./dist/index.js" }, - "./util": { - "types": "./util/dist/index.d.ts", - "import": "./util/dist/index.mjs", - "module": "./util/dist/index.mjs", - "require": "./util/dist/index.js" - }, "./instrumentation": { "types": "./dist/instrumentation/index.d.ts", "import": "./dist/instrumentation/index.mjs", @@ -107,10 +96,6 @@ "module": "./dist/auto-instrumentations/bundler/next.mjs", "require": "./dist/auto-instrumentations/bundler/next.cjs" }, - "./webpack-loader": { - "types": "./dist/auto-instrumentations/bundler/webpack-loader.d.ts", - "require": "./dist/auto-instrumentations/bundler/webpack-loader.cjs" - }, "./esbuild": { "types": "./dist/auto-instrumentations/bundler/esbuild.d.ts", "import": "./dist/auto-instrumentations/bundler/esbuild.mjs", @@ -126,7 +111,6 @@ }, "files": [ "dist/**/*", - "util/dist/**/*", "bin/bt", "scripts/bt-helper.js", "scripts/install.js", @@ -138,7 +122,7 @@ "build": "cross-env NODE_OPTIONS=\"--max-old-space-size=8192\" tsup", "check:typings": "tsc --noEmit && tsc --noEmit -p tsconfig.test.json", "watch": "tsup --watch", - "clean": "rm -r dist/* && rm -r util/dist/*", + "clean": "rm -r dist/*", "docs": "typedoc --options typedoc.json src/node/index.ts", "test": "vitest run --exclude \"src/wrappers/**/*.test.ts\" --exclude \"src/otel/**/*.test.ts\" --exclude \"smoke/**/*.test.ts\" --exclude \"tests/api-compatibility/**\"", "test:core": "pnpm prune && pnpm test", diff --git a/js/smoke/scenarios/browser-main-package/README.md b/js/smoke/scenarios/browser-main-package/README.md index 1b78d3562..58a6f72fb 100644 --- a/js/smoke/scenarios/browser-main-package/README.md +++ b/js/smoke/scenarios/browser-main-package/README.md @@ -1,6 +1,6 @@ # Browser Main Package Smoke Test -This smoke test verifies that the informational message appears when using the browser build from the main `braintrust` package. +This smoke test verifies the curated browser API exposed by the main `braintrust` package. ## What This Tests @@ -10,11 +10,12 @@ When a user imports from the main `braintrust` package in a browser environment: import * as braintrust from "braintrust"; ``` -The bundler (via the `"browser"` field in package.json) will resolve to the browser build (`dist/browser.mjs`), which should: +The package's browser export condition resolves to `dist/browser.mjs`, which should: -1. Show an informational console message suggesting `@braintrust/browser` for optimal use -2. Provide working browser-safe implementations -3. Not include Node.js modules +1. Provide working browser-safe implementations +2. Expose the supported root API +3. Omit removed legacy and internal exports +4. Not include Node.js modules ## Test Structure @@ -44,21 +45,6 @@ pnpm exec playwright test ## What Gets Verified ✓ Import from main package works in browser -✓ Basic functions are available (init, newId, traceable) -✓ Informational message appears in console -✓ Message mentions "@braintrust/browser" package +✓ Supported functions are available (`init`, `flush`) +✓ Legacy exports such as `newId` and `traceable` are absent ✓ No Node.js module errors - -## Expected Console Output - -When the test runs, you should see: - -``` -Braintrust SDK Browser Build -You are using a browser-compatible build from the main package. -For optimal browser support consider: - npm install @braintrust/browser - import * as braintrust from "@braintrust/browser" -``` - -This message guides users toward the optimized `@braintrust/browser` package while ensuring the main package works correctly in browsers. diff --git a/js/smoke/scenarios/browser-main-package/src/browser-message-test.ts b/js/smoke/scenarios/browser-main-package/src/browser-message-test.ts index dd728e468..92f570a50 100644 --- a/js/smoke/scenarios/browser-main-package/src/browser-message-test.ts +++ b/js/smoke/scenarios/browser-main-package/src/browser-message-test.ts @@ -5,8 +5,8 @@ declare global { consoleMessages: string[]; importSuccessful: boolean; hasInit: boolean; - hasNewId: boolean; - hasTraceable: boolean; + hasFlush: boolean; + removedLegacyExports: boolean; }; } } @@ -22,13 +22,14 @@ console.info = (...args: any[]) => { // Import from main package browser export AFTER setting up console capture // This must be done dynamically to ensure console.info is overridden first -const braintrust = await import("braintrust/browser"); +const braintrust = await import("braintrust"); // Test that imports work const importSuccessful = true; const hasInit = typeof braintrust.init === "function"; -const hasNewId = typeof braintrust.newId === "function"; -const hasTraceable = typeof braintrust.traceable === "function"; +const hasFlush = typeof braintrust.flush === "function"; +const removedLegacyExports = + !("newId" in braintrust) && !("traceable" in braintrust); // Store results window.__btBrowserMessageTest = { @@ -36,8 +37,8 @@ window.__btBrowserMessageTest = { consoleMessages: capturedMessages, importSuccessful, hasInit, - hasNewId, - hasTraceable, + hasFlush, + removedLegacyExports, }; // Display results @@ -48,8 +49,8 @@ if (output) {
  • Import successful: ${importSuccessful ? "✓" : "✗"}
  • Has init function: ${hasInit ? "✓" : "✗"}
    • -
  • Has newId function: ${hasNewId ? "✓" : "✗"}
    • -
  • Has traceable function: ${hasTraceable ? "✓" : "✗"}
    • +
  • Has flush function: ${hasFlush ? "✓" : "✗"}
    • +
  • Legacy exports removed: ${removedLegacyExports ? "✓" : "✗"}
  • Console messages captured: ${capturedMessages.length}

Console Messages:

diff --git a/js/smoke/scenarios/browser-main-package/tests/browser-message.test.ts b/js/smoke/scenarios/browser-main-package/tests/browser-message.test.ts index 8c5d9fb7e..3a48a7f60 100644 --- a/js/smoke/scenarios/browser-main-package/tests/browser-message.test.ts +++ b/js/smoke/scenarios/browser-main-package/tests/browser-message.test.ts @@ -1,7 +1,7 @@ import { test, expect } from "@playwright/test"; test.describe("Braintrust Main Package Browser Build", () => { - test("should display informational message when using browser build", async ({ + test("should expose the curated root API in a browser", async ({ page, baseURL, }) => { @@ -55,7 +55,7 @@ test.describe("Braintrust Main Package Browser Build", () => { expect(testResults.completed).toBe(true); expect(testResults.importSuccessful).toBe(true); expect(testResults.hasInit).toBe(true); - expect(testResults.hasNewId).toBe(true); - expect(testResults.hasTraceable).toBe(true); + expect(testResults.hasFlush).toBe(true); + expect(testResults.removedLegacyExports).toBe(true); }); }); diff --git a/js/smoke/scenarios/cloudflare-vite-hono-vite-dev-node-esm/tests/node-esm-import.test.mjs b/js/smoke/scenarios/cloudflare-vite-hono-vite-dev-node-esm/tests/node-esm-import.test.mjs index 848eaef79..a5af1c383 100644 --- a/js/smoke/scenarios/cloudflare-vite-hono-vite-dev-node-esm/tests/node-esm-import.test.mjs +++ b/js/smoke/scenarios/cloudflare-vite-hono-vite-dev-node-esm/tests/node-esm-import.test.mjs @@ -86,7 +86,7 @@ async function testViteDevServerWithNodeEsm() { details: "Vite's dependency pre-bundler cannot handle Nunjucks' use of Object.setPrototypeOf when pre-bundling braintrust/node", recommendation: - "Use 'braintrust/browser' import or configure Vite to exclude Nunjucks from optimization", + "Import 'braintrust' through its browser condition or configure Vite to exclude Nunjucks from optimization", }; } @@ -163,7 +163,7 @@ async function testViteDevServerWithNodeEsm() { "Expected failure: Nunjucks incompatibility with Vite bundler when using braintrust/node. " + "Root cause: Nunjucks uses Object.setPrototypeOf in ways incompatible with Vite's ESM bundler. " + "The Node.js ESM build includes Nunjucks, which causes this error during Vite's dependency pre-bundling. " + - "Recommendation: Use 'braintrust/browser' import or exclude Nunjucks from Vite optimization", + "Recommendation: Import 'braintrust' through its browser condition or exclude Nunjucks from Vite optimization", error: errorStack ? { message: testResult.error, diff --git a/js/smoke/scenarios/cloudflare-vite-hono-vite-dev/README.md b/js/smoke/scenarios/cloudflare-vite-hono-vite-dev/README.md index 2485bd93a..88af0b7e8 100644 --- a/js/smoke/scenarios/cloudflare-vite-hono-vite-dev/README.md +++ b/js/smoke/scenarios/cloudflare-vite-hono-vite-dev/README.md @@ -10,7 +10,7 @@ Tests whether the Braintrust SDK (browser build) can be loaded in Vite's dev ser **Hono framework:** Uses Hono for elegant routing instead of raw Worker API. -**Browser build:** Uses `braintrust/browser` import since Cloudflare Workers don't support Node.js APIs. +**Browser build:** Imports `braintrust`; the package's browser condition selects the browser-safe build because Cloudflare Workers don't support Node.js APIs. ## Test Suite diff --git a/js/smoke/scenarios/cloudflare-vite-hono-wrangler-dev/README.md b/js/smoke/scenarios/cloudflare-vite-hono-wrangler-dev/README.md index 77b51ac01..de82ebdfb 100644 --- a/js/smoke/scenarios/cloudflare-vite-hono-wrangler-dev/README.md +++ b/js/smoke/scenarios/cloudflare-vite-hono-wrangler-dev/README.md @@ -10,7 +10,7 @@ Tests Braintrust SDK in a Cloudflare Workers environment with Vite bundling via **Hono framework:** Uses Hono for elegant routing instead of raw Worker API. -**Browser build:** Uses `braintrust/browser` import since Cloudflare Workers don't support Node.js APIs. +**Browser build:** Imports `braintrust`; the package's browser condition selects the browser-safe build because Cloudflare Workers don't support Node.js APIs. **Wrangler dev:** Tests via Wrangler dev server, which uses Vite to build the worker. diff --git a/js/smoke/scenarios/playwright-browser/src/browser-tests.ts b/js/smoke/scenarios/playwright-browser/src/browser-tests.ts index 403f1abc7..706b13cfd 100644 --- a/js/smoke/scenarios/playwright-browser/src/browser-tests.ts +++ b/js/smoke/scenarios/playwright-browser/src/browser-tests.ts @@ -44,15 +44,6 @@ const harness = createBrowserHarness(output); window.__btBrowserSmokeResults = harness.results; async function runAllTestSuites() { - if (!braintrust._exportsForTestingOnly) { - harness.fail( - "runtime", - "preflight", - new Error("_exportsForTestingOnly not available"), - ); - return; - } - harness.log("=== Running test suites ==="); const { passed, failed, xfail } = await runTests({ diff --git a/js/smoke/shared/src/helpers/register.ts b/js/smoke/shared/src/helpers/register.ts index 9b45d0126..9c0cce707 100644 --- a/js/smoke/shared/src/helpers/register.ts +++ b/js/smoke/shared/src/helpers/register.ts @@ -18,7 +18,10 @@ export interface TestContext { backgroundLogger: BackgroundLogger; } -export type TestFn = (braintrust: BraintrustModule) => Promise; +export type TestFn = ((braintrust: BraintrustModule) => Promise) & { + testName: string; + requiresTestHarness?: boolean; +}; type RegisteredTestFn = ( braintrust: BraintrustModule, @@ -35,24 +38,32 @@ type RegisteredTestFn = ( * - Return a partial TestResult -> merged with pass status * - Throw an error -> fail with error details */ -export function register(name: string, fn: RegisteredTestFn): TestFn { +export function register( + name: string, + fn: RegisteredTestFn, + { requiresTestHarness = false }: { requiresTestHarness?: boolean } = {}, +): TestFn { registeredTests.add(name); - return async (braintrust: BraintrustModule): Promise => { - const testing = braintrust._exportsForTestingOnly as { - setInitialTestState: () => void; - simulateLoginForTests: () => Promise | unknown; - simulateLogoutForTests?: () => Promise | unknown; - useTestBackgroundLogger: () => BackgroundLogger; - clearTestBackgroundLogger: () => void; - }; + const test = async (braintrust: BraintrustModule): Promise => { + const testing = braintrust._exportsForTestingOnly as + | { + setInitialTestState: () => void; + simulateLoginForTests: () => Promise | unknown; + simulateLogoutForTests?: () => Promise | unknown; + useTestBackgroundLogger: () => BackgroundLogger; + clearTestBackgroundLogger: () => void; + } + | undefined; - testing.setInitialTestState(); - await testing.simulateLoginForTests(); - const backgroundLogger = testing.useTestBackgroundLogger(); + testing?.setInitialTestState(); + await testing?.simulateLoginForTests(); + const backgroundLogger = testing?.useTestBackgroundLogger(); try { - const result = await fn(braintrust, { backgroundLogger }); + const result = await fn(braintrust, { + backgroundLogger: backgroundLogger!, + }); if (typeof result === "string") { return { status: "pass", name, message: result }; @@ -71,12 +82,15 @@ export function register(name: string, fn: RegisteredTestFn): TestFn { }, }; } finally { - testing.clearTestBackgroundLogger(); - if (typeof testing.simulateLogoutForTests === "function") { + testing?.clearTestBackgroundLogger(); + if (typeof testing?.simulateLogoutForTests === "function") { await testing.simulateLogoutForTests(); } } }; + test.testName = name; + test.requiresTestHarness = requiresTestHarness; + return test; } /** @@ -146,9 +160,21 @@ export async function runTests({ skipCoverage = false, }: RunTestsOptions): Promise { const results: TestResult[] = []; + const testModule: BraintrustModule = { + ...braintrust, + __publicModule: braintrust, + }; for (const test of tests) { - results.push(await test(braintrust)); + if (!testModule._exportsForTestingOnly && test.requiresTestHarness) { + results.push({ + status: "xfail", + name: test.testName, + message: "Requires the private SDK test harness", + }); + } else { + results.push(await test(testModule)); + } } if (!skipCoverage) { @@ -186,7 +212,7 @@ export function expectFailure( predicate: ErrorPredicate, reason: string, ): TestFn { - return async (braintrust: BraintrustModule): Promise => { + const wrapped = async (braintrust: BraintrustModule): Promise => { const result = await test(braintrust); if (result.status === "fail" && result.error) { @@ -200,4 +226,7 @@ export function expectFailure( } return result; }; + wrapped.testName = test.testName; + wrapped.requiresTestHarness = test.requiresTestHarness; + return wrapped; } diff --git a/js/smoke/shared/src/suites/basic-logging.ts b/js/smoke/shared/src/suites/basic-logging.ts index 9bd174022..1a92af9d4 100644 --- a/js/smoke/shared/src/suites/basic-logging.ts +++ b/js/smoke/shared/src/suites/basic-logging.ts @@ -47,6 +47,7 @@ export const testBasicSpanLogging = register( return "Basic span logging test passed"; }, + { requiresTestHarness: true }, ); export const testMultipleSpans = register( @@ -78,6 +79,7 @@ export const testMultipleSpans = register( return `Multiple spans test passed (${events.length} events captured)`; }, + { requiresTestHarness: true }, ); export const testDirectLogging = register( @@ -102,6 +104,7 @@ export const testDirectLogging = register( return "Direct logging test passed"; }, + { requiresTestHarness: true }, ); export const testJSONAttachment = register( @@ -159,6 +162,7 @@ export const testJSONAttachment = register( return "JSONAttachment test passed"; }, + { requiresTestHarness: true }, ); export const testAsyncLocalStorageTraced = register( @@ -221,6 +225,7 @@ export const testAsyncLocalStorageTraced = register( ); } }, + { requiresTestHarness: true }, ); export const testNestedTraced = register( @@ -301,6 +306,7 @@ export const testNestedTraced = register( return "ALS not available in this environment, test skipped"; } }, + { requiresTestHarness: true }, ); export const testCurrentSpan = register( @@ -355,4 +361,5 @@ export const testCurrentSpan = register( throw new Error("currentSpan() returned a span ID that was not logged"); } }, + { requiresTestHarness: true }, ); diff --git a/js/smoke/shared/src/suites/import-verification.ts b/js/smoke/shared/src/suites/import-verification.ts index cf422006b..220ff39b5 100644 --- a/js/smoke/shared/src/suites/import-verification.ts +++ b/js/smoke/shared/src/suites/import-verification.ts @@ -24,6 +24,7 @@ import { register, type TestFn } from "../helpers/register"; * Interface for the Braintrust module based on exports.ts */ export interface BraintrustModule { + __publicModule?: BraintrustModule; // Core logging (REQUIRED) initLogger?: unknown; Logger?: unknown; @@ -87,10 +88,8 @@ export interface BraintrustModule { projects?: unknown; PromptBuilder?: unknown; - // ID Generation (REQUIRED) + // Legacy private smoke-test harness fields IDGenerator?: unknown; - - // Testing (REQUIRED) _exportsForTestingOnly?: unknown; // State management (REQUIRED) @@ -126,13 +125,30 @@ export const testCoreLoggingExports = register( assertDefined(module.startSpan, "startSpan must exist"); assertType(module.startSpan, "function", "startSpan must be a function"); - assertDefined(module.log, "log must exist"); - assertType(module.log, "function", "log must be a function"); - assertDefined(module.flush, "flush must exist"); assertType(module.flush, "function", "flush must be a function"); - return "Core logging exports verified (7 exports)"; + const publicModule = module.__publicModule ?? module; + for (const removed of [ + "default", + "log", + "summarize", + "initExperiment", + "traceable", + "EvalResultWithSummary", + "SpanImpl", + "_exportsForTestingOnly", + "AttachmentReference", + "promptDefinitionSchema", + "braintrustEveInstrumentation", + "braintrustFlueInstrumentation", + ]) { + if (removed in publicModule) { + throw new Error(`${removed} must not be publicly exported`); + } + } + + return "Core logging exports and removals verified"; }, ); @@ -176,12 +192,8 @@ export const testPromptExports = register( export const testExperimentExports = register( "testExperimentExports", async (module) => { - assertDefined(module.initExperiment, "initExperiment must exist"); - assertType( - module.initExperiment, - "function", - "initExperiment must be a function", - ); + assertDefined(module.init, "init must exist"); + assertType(module.init, "function", "init must be a function"); assertDefined(module.Experiment, "Experiment must exist"); assertType( @@ -205,16 +217,6 @@ export const testEvalExports = register("testEvalExports", async (module) => { assertDefined(module.Eval, "Eval must exist"); assertType(module.Eval, "function", "Eval must be a function"); - assertDefined( - module.EvalResultWithSummary, - "EvalResultWithSummary must exist", - ); - assertType( - module.EvalResultWithSummary, - "function", - "EvalResultWithSummary must be a function/class", - ); - assertDefined(module.Reporter, "Reporter must exist"); assertType(module.Reporter, "function", "Reporter must be a function"); @@ -228,7 +230,7 @@ export const testEvalExports = register("testEvalExports", async (module) => { "defaultErrorScoreHandler must be a function", ); - return "Eval exports verified (4 runtime exports)"; + return "Eval exports verified (3 runtime exports)"; }); export const testTracingExports = register( @@ -237,9 +239,6 @@ export const testTracingExports = register( assertDefined(module.traced, "traced must exist"); assertType(module.traced, "function", "traced must be a function"); - assertDefined(module.traceable, "traceable must exist"); - assertType(module.traceable, "function", "traceable must be a function"); - assertDefined(module.wrapTraced, "wrapTraced must exist"); assertType(module.wrapTraced, "function", "wrapTraced must be a function"); @@ -260,7 +259,7 @@ export const testTracingExports = register( "withCurrent must be a function", ); - return "Tracing exports verified (6 exports)"; + return "Tracing exports verified (5 exports)"; }, ); @@ -276,7 +275,6 @@ export const testClientWrapperExports = register( "wrapGoogleGenAI", "wrapOpenRouter", "wrapAISDK", - "wrapMastraAgent", "wrapClaudeAgentSDK", ]; @@ -312,13 +310,7 @@ export const testUtilityExports = register( "Attachment must be a function/class", ); - assertDefined(module.newId, "newId must exist"); - assertType(module.newId, "function", "newId must be a function"); - - assertDefined(module.permalink, "permalink must exist"); - assertType(module.permalink, "function", "permalink must be a function"); - - return "Utility exports verified (4 exports)"; + return "Attachment exports verified (2 exports)"; }, ); @@ -342,20 +334,10 @@ export const testFunctionExports = register( export const testFramework2Exports = register( "testFramework2Exports", async (module) => { - assertDefined(module.Project, "Project must exist"); - assertType(module.Project, "function", "Project must be a function/class"); - assertDefined(module.projects, "projects must exist"); assertType(module.projects, "object", "projects must be an object"); - assertDefined(module.PromptBuilder, "PromptBuilder must exist"); - assertType( - module.PromptBuilder, - "function", - "PromptBuilder must be a function/class", - ); - - return "Framework2 exports verified (3 exports)"; + return "Project facade export verified"; }, ); @@ -369,8 +351,9 @@ export const testIDGeneratorExports = register( "IDGenerator must be a function/class", ); - return "ID generator exports verified (1 export)"; + return "Internal ID generator export verified"; }, + { requiresTestHarness: true }, ); export const testTestingExports = register( @@ -386,8 +369,9 @@ export const testTestingExports = register( "_exportsForTestingOnly must be an object", ); - return "Testing exports verified (1 export)"; + return "Internal testing export verified"; }, + { requiresTestHarness: true }, ); export const testStateManagementExports = register( @@ -415,34 +399,39 @@ export function testBuildResolution( | "workerd" | "node", ): TestFn { - return register("testBuildResolution", async (module) => { - const { buildType: detectedBuild, buildDetails } = detectBuildType(module); - const detectedFormat = detectModuleFormat(); - - if (detectedBuild === "unknown") { - throw new Error( - `Build type is unknown - configureBrowser() or configureNode() was not called. ${buildDetails || ""}`, - ); - } + return register( + "testBuildResolution", + async (module) => { + const { buildType: detectedBuild, buildDetails } = + detectBuildType(module); + const detectedFormat = detectModuleFormat(); + + if (detectedBuild === "unknown") { + throw new Error( + `Build type is unknown - configureBrowser() or configureNode() was not called. ${buildDetails || ""}`, + ); + } - if (expectedBuildType && detectedBuild !== expectedBuildType) { - throw new Error( - `Expected build type "${expectedBuildType}" but detected "${detectedBuild}"`, - ); - } + if (expectedBuildType && detectedBuild !== expectedBuildType) { + throw new Error( + `Expected build type "${expectedBuildType}" but detected "${detectedBuild}"`, + ); + } - const parts: string[] = [`Detected ${detectedBuild} build`]; + const parts: string[] = [`Detected ${detectedBuild} build`]; - if (expectedBuildType) { - parts.push(`(expected: ${expectedBuildType})`); - } + if (expectedBuildType) { + parts.push(`(expected: ${expectedBuildType})`); + } - if (detectedFormat !== "unknown") { - parts.push(`${detectedFormat} format`); - } + if (detectedFormat !== "unknown") { + parts.push(`${detectedFormat} format`); + } - return parts.join(", "); - }); + return parts.join(", "); + }, + { requiresTestHarness: true }, + ); } function detectBuildType(module: BraintrustModule): { diff --git a/js/src/auto-instrumentations/bundler/esbuild.ts b/js/src/auto-instrumentations/bundler/esbuild.ts index dc1443ad8..9e96a2c90 100644 --- a/js/src/auto-instrumentations/bundler/esbuild.ts +++ b/js/src/auto-instrumentations/bundler/esbuild.ts @@ -7,10 +7,3 @@ export function braintrustEsbuildPlugin( ): EsbuildPlugin { return unplugin.esbuild(options); } - -export type EsbuildPluginOptions = BundlerPluginOptions; - -/** - * @deprecated Use {@link braintrustEsbuildPlugin} instead. - */ -export const esbuildPlugin = unplugin.esbuild; diff --git a/js/src/auto-instrumentations/bundler/next.ts b/js/src/auto-instrumentations/bundler/next.ts index 774a4ab98..286be8f99 100644 --- a/js/src/auto-instrumentations/bundler/next.ts +++ b/js/src/auto-instrumentations/bundler/next.ts @@ -1,6 +1,6 @@ import { createRequire } from "node:module"; -import { join } from "node:path"; -import { webpackPlugin } from "./webpack"; +import { dirname, join } from "node:path"; +import { braintrustWebpackPlugin } from "./webpack"; type MaybePromise = T | Promise; type NextConfigFunction = ( @@ -144,7 +144,7 @@ function wrapWebpackConfig( : "server" : "client"; - const plugin = webpackPlugin({ + const plugin = braintrustWebpackPlugin({ browser: runtime === "client" || runtime === "edge", }); @@ -233,11 +233,11 @@ function addBraintrustTurbopackRule( } function getWebpackLoaderPath(): string { - try { - return requireFromProject.resolve("braintrust/webpack-loader"); - } catch { - return "braintrust/webpack-loader"; - } + const packageJsonPath = requireFromProject.resolve("braintrust/package.json"); + return join( + dirname(packageJsonPath), + "dist/auto-instrumentations/bundler/webpack-loader.cjs", + ); } function getNextMajorVersion(): number | undefined { diff --git a/js/src/auto-instrumentations/bundler/plugin.ts b/js/src/auto-instrumentations/bundler/plugin.ts index 47562db0d..98361aaea 100644 --- a/js/src/auto-instrumentations/bundler/plugin.ts +++ b/js/src/auto-instrumentations/bundler/plugin.ts @@ -24,17 +24,6 @@ export interface BundlerPluginOptions { * @default false */ browser?: boolean; - - /** - * Marks transformed source as targeting a browser or edge-like environment. - * - * This retains the previous browser-target behavior of the option. Global - * instrumentation hooks are runtime-independent, so no diagnostics-channel - * compatibility shim is injected. - * - * @deprecated Use `browser` instead. - */ - useDiagnosticChannelCompatShim?: boolean; } /** @@ -55,8 +44,7 @@ function getModuleVersion(basedir: string): string | undefined { } export const unplugin = createUnplugin((options = {}) => { - const browser = - options.browser ?? options.useDiagnosticChannelCompatShim ?? false; + const browser = options.browser ?? false; const allInstrumentations = getDefaultAutoInstrumentationConfigs( options.instrumentations, ); diff --git a/js/src/auto-instrumentations/bundler/rollup.ts b/js/src/auto-instrumentations/bundler/rollup.ts index fc93bb614..a27de3589 100644 --- a/js/src/auto-instrumentations/bundler/rollup.ts +++ b/js/src/auto-instrumentations/bundler/rollup.ts @@ -7,10 +7,3 @@ export function braintrustRollupPlugin( ): RollupPlugin | RollupPlugin[] { return unplugin.rollup(options); } - -export type RollupPluginOptions = BundlerPluginOptions; - -/** - * @deprecated Use {@link braintrustRollupPlugin} instead. - */ -export const rollupPlugin = unplugin.rollup; diff --git a/js/src/auto-instrumentations/bundler/vite.ts b/js/src/auto-instrumentations/bundler/vite.ts index 297949eff..51f0e66d2 100644 --- a/js/src/auto-instrumentations/bundler/vite.ts +++ b/js/src/auto-instrumentations/bundler/vite.ts @@ -39,10 +39,3 @@ export function braintrustVitePlugin( ...(Array.isArray(transformPlugin) ? transformPlugin : [transformPlugin]), ]; } - -export type VitePluginOptions = BundlerPluginOptions; - -/** - * @deprecated Use {@link braintrustVitePlugin} instead. - */ -export const vitePlugin = unplugin.vite; diff --git a/js/src/auto-instrumentations/bundler/webpack-loader.ts b/js/src/auto-instrumentations/bundler/webpack-loader.ts index 211cd31a5..80874304e 100644 --- a/js/src/auto-instrumentations/bundler/webpack-loader.ts +++ b/js/src/auto-instrumentations/bundler/webpack-loader.ts @@ -4,21 +4,8 @@ * This is a webpack loader (not a plugin) for compatibility with tools that only support loaders, * such as Next.js Turbopack. * - * Usage in next.config.js / next.config.ts: - * ```javascript - * const nextConfig: NextConfig = { - * turbopack: { - * rules: { - * // Apply the loader to all JS/MJS/CJS files from node_modules. - * // condition: "foreign" restricts the rule to third-party packages only. - * "*.{js,mjs,cjs}": { - * condition: "foreign", - * loaders: [{ loader: require.resolve("braintrust/webpack-loader") }], - * }, - * }, - * }, - * }; - * ``` + * The `braintrust/next` entrypoint resolves this implementation directly when + * configuring Turbopack; it is not a user-facing package entrypoint. */ import { create } from "../orchestrion-js"; @@ -130,7 +117,7 @@ function codeTransformerLoader( modulePath: normalizedModulePath, source: code, format: isModule ? "esm" : "cjs", - browser: options.browser ?? options.useDiagnosticChannelCompatShim ?? false, + browser: options.browser ?? false, }); if (patched !== null) { return callback(null, patched); diff --git a/js/src/auto-instrumentations/bundler/webpack.ts b/js/src/auto-instrumentations/bundler/webpack.ts index b81da8456..0617ee9bc 100644 --- a/js/src/auto-instrumentations/bundler/webpack.ts +++ b/js/src/auto-instrumentations/bundler/webpack.ts @@ -7,10 +7,3 @@ export function braintrustWebpackPlugin( ): WebpackPluginInstance { return unplugin.webpack(options); } - -export type WebpackPluginOptions = BundlerPluginOptions; - -/** - * @deprecated Use {@link braintrustWebpackPlugin} instead. - */ -export const webpackPlugin = unplugin.webpack; diff --git a/js/src/auto-instrumentations/configs/all.ts b/js/src/auto-instrumentations/configs/all.ts index 3b42cb11a..c576eaccf 100644 --- a/js/src/auto-instrumentations/configs/all.ts +++ b/js/src/auto-instrumentations/configs/all.ts @@ -47,11 +47,11 @@ const defaultInstrumentationConfigGroups: readonly InstrumentationConfigGroup[] }, { integrations: ["anthropic"], configs: anthropicConfigs }, { - integrations: ["bedrock", "awsBedrock", "awsBedrockRuntime"], + integrations: ["awsBedrockRuntime"], configs: bedrockRuntimeConfigs, }, { - integrations: ["aisdk", "vercel"], + integrations: ["aisdk"], configs: aiSDKConfigs, }, { @@ -70,13 +70,13 @@ const defaultInstrumentationConfigGroups: readonly InstrumentationConfigGroup[] integrations: ["cloudflareThink"], configs: cloudflareThinkConfigs, }, - { integrations: ["cursor", "cursorSDK"], configs: cursorSDKConfigs }, + { integrations: ["cursorSDK"], configs: cursorSDKConfigs }, { integrations: ["openAIAgents"], configs: openAIAgentsCoreConfigs, }, { - integrations: ["google", "googleGenAI"], + integrations: ["googleGenAI"], configs: googleGenAIConfigs, }, { diff --git a/js/src/browser/index.ts b/js/src/browser/index.ts index f478444b7..dfbca964f 100644 --- a/js/src/browser/index.ts +++ b/js/src/browser/index.ts @@ -14,4 +14,3 @@ import { configureBrowser } from "./config"; configureBrowser(); export * from "../exports"; -export * as default from "../exports"; diff --git a/js/src/edge-light/index.ts b/js/src/edge-light/index.ts index 4c6ef7bbf..800419f2a 100644 --- a/js/src/edge-light/index.ts +++ b/js/src/edge-light/index.ts @@ -10,4 +10,3 @@ import { configureEdgeLight } from "./config"; configureEdgeLight(); export * from "../exports"; -export * as default from "../exports"; diff --git a/js/src/edge-runtime-bootstrap.test.ts b/js/src/edge-runtime-bootstrap.test.ts index 39f772620..7cfce0fa8 100644 --- a/js/src/edge-runtime-bootstrap.test.ts +++ b/js/src/edge-runtime-bootstrap.test.ts @@ -67,19 +67,19 @@ describe.each([ }); const braintrust = await import(entrypoint); + const { _exportsForTestingOnly } = await import("./logger"); - braintrust._exportsForTestingOnly.setInitialTestState(); - await braintrust._exportsForTestingOnly.simulateLoginForTests(); + _exportsForTestingOnly.setInitialTestState(); + await _exportsForTestingOnly.simulateLoginForTests(); - const backgroundLogger = - braintrust._exportsForTestingOnly.useTestBackgroundLogger(); + const backgroundLogger = _exportsForTestingOnly.useTestBackgroundLogger(); const logger = braintrust.initLogger({ projectId: "test-project-id", projectName, }); const root = logger.startSpan({ name: "root", type: "task" }); - const parent = await root.export(); + const parent = braintrust.extractTraceContextFromHeaders(root.inject())!; const result = await braintrust.traced( async (span: unknown) => { @@ -113,12 +113,12 @@ describe.each([ test("wrapAISDK logs spans in edge runtimes", async () => { const braintrust = await import(entrypoint); + const { _exportsForTestingOnly } = await import("./logger"); - braintrust._exportsForTestingOnly.setInitialTestState(); - await braintrust._exportsForTestingOnly.simulateLoginForTests(); + _exportsForTestingOnly.setInitialTestState(); + await _exportsForTestingOnly.simulateLoginForTests(); - const backgroundLogger = - braintrust._exportsForTestingOnly.useTestBackgroundLogger(); + const backgroundLogger = _exportsForTestingOnly.useTestBackgroundLogger(); braintrust.initLogger({ projectId: "test-project-id", projectName, diff --git a/js/src/exports.ts b/js/src/exports.ts index ffce08390..d08f492e5 100644 --- a/js/src/exports.ts +++ b/js/src/exports.ts @@ -1,210 +1,79 @@ -import type { z } from "zod/v3"; -import { AttachmentReference as attachmentReferenceSchema } from "./generated_types"; -import type { AttachmentReferenceType } from "./generated_plain_types"; +/** + * The public `braintrust` package surface. + * + * Keep this file as an explicit allowlist. Implementation details should stay + * private; integrations belong on narrow, intentional extension points. + */ export type { - AnyDataset, - AttachmentParams, - BackgroundLoggerOpts, - BaseMetadata, - ChatPrompt, - CompiledPrompt, - CompiledPromptParams, - CompletionPrompt, ContextParentSpanIds, - DatasetRestorePreviewResult, - DatasetRestoreResult, - DatasetSnapshot, - DataSummary, - DatasetSummary, - DefaultMetadataType, - DefaultPromptArgs, - EndSpanArgs, - EvalCase, - ExperimentSummary, - Exportable, - ExternalAttachmentParams, - FullInitDatasetOptions, - FullInitOptions, - FullLoginOptions, - InitDatasetOptions, - InitLoggerOptions, - InitOptions, - LoadPromptOptions, - LoginOptions, - Logs3OverflowInputRow, - Logs3OverflowUpload, - MetricSummary, - ObjectMetadata, + CurrentSpanStore, PropagationContext, - PromptRowWithId, - ScoreSummary, - SerializedBraintrustState, - SetCurrentArg, Span, StartSpanArgs, - WithTransactionId, } from "./logger"; - -export type { - SandboxConfig, - RegisterSandboxOptions, - RegisterSandboxResult, -} from "./sandbox"; - export { Attachment, BaseAttachment, - BRAINTRUST_CURRENT_SPAN_STORE, BraintrustState, - ContextManager, - CurrentSpanStore, - DEFAULT_FETCH_BATCH_SIZE, - DEFAULT_MAX_REQUEST_SIZE, Dataset, - ObjectFetcher, - ERR_PERMALINK, Experiment, ExternalAttachment, - FailedHTTPResponse, JSONAttachment, - LOGS3_OVERFLOW_REFERENCE_TYPE, Logger, - LoginInvalidOrgError, NOOP_SPAN, - NOOP_SPAN_PERMALINK, - NoopSpan, Prompt, ReadonlyAttachment, ReadonlyExperiment, - SpanImpl, - TestBackgroundLogger, - _exportsForTestingOnly, - _internalGetGlobalState, - _internalSetInitialState, - constructLogs3OverflowRequest, currentExperiment, currentLogger, currentSpan, - deepCopyEvent, - deserializePlainStringAsJSON, + configureContextManager, + ContextManager, extractTraceContextFromHeaders, flush, - getContextManager, getPromptVersions, - getSpanParentObject, init, injectTraceContext, initDataset, - initExperiment, initLogger, loadParameters, loadPrompt, - log, logError, login, loginToState, - logs3OverflowUploadSchema, - newId, - permalink, - pickLogs3OverflowObjectIds, - uploadLogs3OverflowPayload, - utf8ByteLength, - renderMessage, - renderPromptParams, setFetch, setMaskingFunction, - spanComponentsToObjectId, startSpan, - summarize, - traceable, traced, updateSpan, withCurrent, - withDataset, - withExperiment, withParent, wrapTraced, - registerOtelFlush, } from "./logger"; export { registerSandbox } from "./sandbox"; -// Internal isomorph layer for platform-specific implementations -import _internalIso from "./isomorph"; -export { _internalIso }; - +export type { TemplateRendererPlugin } from "./template/registry"; +export { registerTemplatePlugin } from "./template/registry"; export { isTemplateFormat, parseTemplateFormat, renderTemplateContent, } from "./template/renderer"; -export type { TemplateFormat } from "./template/registry"; -export type { - TemplateRenderer, - TemplateRendererPlugin, -} from "./template/registry"; -export { - registerTemplatePlugin, - getTemplateRenderer, - templateRegistry, -} from "./template/registry"; - -export type { InvokeFunctionArgs, InvokeReturn } from "./functions/invoke"; export { initFunction, invoke } from "./functions/invoke"; +export { BraintrustStream } from "./functions/stream"; -export type { BraintrustStreamChunk } from "./functions/stream"; -export { - BraintrustStream, - braintrustStreamChunkSchema, - createFinalValuePassThroughStream, - devNullWritableStream, -} from "./functions/stream"; - -export { - IDGenerator, - UUIDGenerator, - OTELIDGenerator, - getIdGenerator, -} from "./id-gen"; - -export { - TRACEPARENT_HEADER, - TRACESTATE_HEADER, - BAGGAGE_HEADER, - BRAINTRUST_PARENT_KEY, -} from "./propagation"; -export type { - ParsedTraceparent, - PropagatedState, - TraceContextCarrier, - TraceContextHeaders, -} from "./propagation"; - -export { - LEGACY_CACHED_HEADER, - X_CACHED_HEADER, - parseCachedHeader, - wrapOpenAI, - wrapOpenAIv4, -} from "./wrappers/oai"; - +export { wrapOpenAI } from "./wrappers/oai"; export { braintrustAISDKTelemetry, wrapAISDK, wrapAgentClass, - BraintrustMiddleware, - wrapAISDKModel, } from "./wrappers/ai-sdk"; -export { braintrustEveInstrumentation } from "./instrumentation"; export { collectAnthropicSession } from "./wrappers/anthropic-session-collector"; export { wrapAnthropic } from "./wrappers/anthropic"; -export { - BraintrustObservabilityExporter, - wrapMastraAgent, -} from "./wrappers/mastra"; -export type { MastraObservabilityExporter } from "./wrappers/mastra"; +export { BraintrustObservabilityExporter } from "./wrappers/mastra"; export { wrapClaudeAgentSDK } from "./wrappers/claude-agent-sdk/claude-agent-sdk"; export { wrapCloudflareThink } from "./wrappers/cloudflare-think"; export { wrapOpenAICodexSDK } from "./wrappers/openai-codex"; @@ -234,37 +103,24 @@ export { } from "./wrappers/langsmith"; export { wrapVitest } from "./wrappers/vitest"; export { initNodeTestSuite } from "./wrappers/node-test"; -export { - BRAINTRUST_LANGCHAIN_CALLBACK_HANDLER_NAME, - BraintrustLangChainCallbackHandler, -} from "./wrappers/langchain/callback-handler"; +export { BraintrustLangChainCallbackHandler } from "./wrappers/langchain/callback-handler"; export type { LangChainCallbackHandlerOptions } from "./vendor-sdk-types/langchain"; -export * as graph from "./graph-framework"; - export type { Evaluator, EvalTask, - EvalHooks, - EvalResult, - EvalScorerArgs, EvalScorer, EvalClassifier, - EvaluatorDef, ReporterBody, - SpanContext, } from "./framework"; - export { BaseExperiment, Eval, - EvalResultWithSummary, Reporter, defaultErrorScoreHandler, } from "./framework"; export type { DurableEvalStore } from "./durable-eval"; - export { BatchScorer, BatchTask, @@ -274,61 +130,8 @@ export { } from "./durable-eval"; export { agentAssertionScorer } from "./agent-assertions"; - export { DatasetPipeline } from "./dataset-pipeline"; +export { projects } from "./framework2"; -export type { - CodeOpts, - CreateProjectOpts, - PromptOpts, - ScorerOpts, -} from "./framework2"; - -export { - CodeFunction, - CodePrompt, - Project, - PromptBuilder, - ScorerBuilder, - ToolBuilder, - projects, -} from "./framework2"; - -export { - promptContentsSchema, - promptDefinitionSchema, - promptDefinitionToPromptData, - promptDefinitionWithToolsSchema, - PromptContents, - PromptDefinition, - PromptDefinitionWithTools, -} from "./prompt-schemas"; - -export type { Trace, SpanData, GetThreadOptions } from "./trace"; -export { SpanFetcher, CachedSpanFetcher, LocalTrace } from "./trace"; - -export type { - ParentExperimentIds, - ParentProjectLogIds, - IdField, - InputField, - OtherExperimentLogFields, - ExperimentLogPartialArgs, - ExperimentLogFullArgs, - LogFeedbackFullArgs, - LogCommentFullArgs, - CommentEvent, - DatasetRecord, -} from "../util"; - -export { addAzureBlobHeaders, LazyValue } from "./util"; - -export const AttachmentReference: z.ZodType = - attachmentReferenceSchema; - -export type { EvalParameters } from "./eval-parameters"; - -// Auto-instrumentation configuration -export { configureInstrumentation } from "./instrumentation"; -export { braintrustFlueInstrumentation } from "./instrumentation"; -export type { InstrumentationConfig } from "./instrumentation"; +export type { Trace } from "./trace"; +export { LocalTrace } from "./trace"; diff --git a/js/src/framework.test.ts b/js/src/framework.test.ts index cb905aad7..a61b578c2 100644 --- a/js/src/framework.test.ts +++ b/js/src/framework.test.ts @@ -45,10 +45,8 @@ test("meta (write) is passed to task", async () => { projectName: "proj", evalName: "eval", data: [{ input: 1, metadata }], - task: async (input: number, { meta }) => { - meta({ - foo: "barbar", - }); + task: async (input: number, { metadata }) => { + metadata.foo = "barbar"; return input * 2; }, scores: [], diff --git a/js/src/framework.ts b/js/src/framework.ts index 540b95dc6..9d2bef9d3 100644 --- a/js/src/framework.ts +++ b/js/src/framework.ts @@ -42,7 +42,7 @@ import { startSpan, traced, withCurrent, - withParent, + _internalWithParent, _internalGetGlobalState, } from "./logger"; import type { ProgressReporter } from "./reporters/types"; @@ -131,10 +131,6 @@ export interface EvalHooks< Metadata extends BaseMetadata, Parameters extends EvalParameters, > { - /** - * @deprecated Use `metadata` instead. - */ - meta: (info: Metadata) => void; /** * The metadata object for the current evaluation. You can mutate this object to add or remove metadata. */ @@ -371,37 +367,14 @@ export interface Evaluator< flushBeforeScoring?: boolean; } -export class EvalResultWithSummary< +export interface EvalResultWithSummary< Input, Output, Expected, Metadata extends BaseMetadata = DefaultMetadataType, > { - constructor( - public summary: ExperimentSummary, - public results: EvalResult[], - ) {} - - /** - * @deprecated Use `summary` instead. - */ - toString(): string { - return JSON.stringify(this.summary); - } - - [Symbol.for("nodejs.util.inspect.custom")](): string { - return `EvalResultWithSummary(summary="...", results=[...])`; - } - - toJSON(): { - summary: ExperimentSummary; - results: EvalResult[]; - } { - return { - summary: this.summary, - results: this.results, - }; - } + summary: ExperimentSummary; + results: EvalResult[]; } export type { ReporterBody } from "./reporters/types"; @@ -552,7 +525,7 @@ export function callEvaluatorData< }; } -export type SpanContext = { +type SpanContext = { currentSpan: typeof currentSpan; startSpan: typeof startSpan; withCurrent: typeof withCurrent; @@ -784,15 +757,15 @@ export async function Eval< _initializeSpanContext(); // Better to return this empty object than have an annoying-to-use signature - return new EvalResultWithSummary( - { + return { + summary: { scores: {}, metrics: {}, projectName: "", experimentName: "", }, - [], - ); + results: [], + }; } const progressReporter = options.progress ?? new SimpleProgressReporter(); @@ -840,7 +813,7 @@ export async function Eval< const enableCache = options.enableCache ?? true; let ret; if (options.parent) { - ret = await withParent( + ret = await _internalWithParent( options.parent, () => runEvaluator( @@ -932,9 +905,6 @@ export async function _internalRunEvaluatorTask( ...("metadata" in datum ? datum.metadata : {}), }; const hooks: EvalHooks, EvalParameters> = { - meta(value) { - Object.assign(metadata, value); - }, metadata, expected: "expected" in datum ? datum.expected : undefined, span, @@ -1525,12 +1495,14 @@ async function runEvaluatorInternal( if (!experiment) { // This will almost always be a no-op span, but it means that if the Eval // is run in the context of a different type of span, it will be logged. + const { parent: _ignoredParent, ...spanEvent } = baseEvent; return await traced(callback, { - ...baseEvent, + ...spanEvent, state: evaluator.state, }); } else { - const result = await experiment.traced(callback, baseEvent); + const { parent: _ignoredParent, ...spanEvent } = baseEvent; + const result = await experiment.traced(callback, spanEvent); // Flush logs to provide backpressure and prevent memory accumulation // when maxConcurrency is set. Only flush when pending data exceeds the // byte threshold, avoiding excessive sequential round-trips for small @@ -1675,10 +1647,10 @@ async function runEvaluatorInternal( localScoreAccumulator ?? undefined, ); - return new EvalResultWithSummary( + return { summary, - collectResults ? collectedResults : [], - ); + results: collectResults ? collectedResults : [], + }; } finally { // Clean up disk-based span cache after eval completes and stop caching // Only if it was enabled diff --git a/js/src/framework2.ts b/js/src/framework2.ts index ab4ee52c5..ed1eef15f 100644 --- a/js/src/framework2.ts +++ b/js/src/framework2.ts @@ -41,7 +41,7 @@ const currentFilename = type NameOrId = { name: string } | { id: string }; -export type CreateProjectOpts = NameOrId; +type CreateProjectOpts = NameOrId; class ProjectBuilder { create(opts: CreateProjectOpts) { return new Project(opts); @@ -49,7 +49,7 @@ class ProjectBuilder { } export const projects = new ProjectBuilder(); -export class Project { +class Project { public readonly name?: string; public readonly id?: string; public tools: ToolBuilder; @@ -142,7 +142,7 @@ export class Project { } } -export class ToolBuilder { +class ToolBuilder { private taskCounter = 0; constructor(private readonly project: Project) {} @@ -212,7 +212,7 @@ export class ToolBuilder { } } -export class ScorerBuilder { +class ScorerBuilder { private taskCounter = 0; constructor(private readonly project: Project) {} @@ -335,7 +335,7 @@ type Schema = Partial<{ returns: z.ZodSchema; }>; -export type CodeOpts< +type CodeOpts< Params, Returns, Fn extends GenericFunction, @@ -376,7 +376,7 @@ type ScorerOptsUnion< | CodeOpts>, Returns, Fn> | ScorerPromptOpts; -export type ScorerOpts< +type ScorerOpts< Output, Input, Params, @@ -561,7 +561,7 @@ interface PromptNoTrace { noTrace: boolean; } -export type PromptOpts< +type PromptOpts< HasId extends boolean, HasVersion extends boolean, HasTools extends boolean = true, @@ -575,7 +575,7 @@ export type PromptOpts< (HasNoTrace extends true ? Partial : {}) & PromptDefinition; -export class PromptBuilder { +class PromptBuilder { constructor(private readonly project: Project) {} public create< diff --git a/js/src/functions/invoke.ts b/js/src/functions/invoke.ts index 82fd40de4..1cfe9a5b3 100644 --- a/js/src/functions/invoke.ts +++ b/js/src/functions/invoke.ts @@ -18,11 +18,7 @@ import { z } from "zod/v3"; /** * Arguments for the `invoke` function. */ -export interface InvokeFunctionArgs< - Input, - Output, - Stream extends boolean = false, -> { +interface InvokeFunctionArgs { // These parameters are duplicated from FunctionId, so that we can document them. /** @@ -136,7 +132,7 @@ export interface InvokeFunctionArgs< * if `stream` is true, otherwise returns the output of the function using the Zod schema's * type if present. */ -export type InvokeReturn = Stream extends true +type InvokeReturn = Stream extends true ? BraintrustStream : Output; diff --git a/js/src/functions/stream.ts b/js/src/functions/stream.ts index 392e6b337..901891119 100644 --- a/js/src/functions/stream.ts +++ b/js/src/functions/stream.ts @@ -30,41 +30,40 @@ export type BraintrustStreamChunk = | { type: "start"; data: string } | { type: "done"; data: string }; -export const braintrustStreamChunkSchema: z.ZodType = - z.union([ - z.object({ - type: z.literal("text_delta"), - data: z.string(), - }), - z.object({ - type: z.literal("reasoning_delta"), - data: z.string(), - }), - z.object({ - type: z.literal("json_delta"), - data: z.string(), - }), - z.object({ - type: z.literal("error"), - data: z.string(), - }), - z.object({ - type: z.literal("console"), - data: sseConsoleEventDataSchema, - }), - z.object({ - type: z.literal("progress"), - data: sseProgressEventDataSchema, - }), - z.object({ - type: z.literal("start"), - data: z.string(), - }), - z.object({ - type: z.literal("done"), - data: z.string(), - }), - ]); +const braintrustStreamChunkSchema: z.ZodType = z.union([ + z.object({ + type: z.literal("text_delta"), + data: z.string(), + }), + z.object({ + type: z.literal("reasoning_delta"), + data: z.string(), + }), + z.object({ + type: z.literal("json_delta"), + data: z.string(), + }), + z.object({ + type: z.literal("error"), + data: z.string(), + }), + z.object({ + type: z.literal("console"), + data: sseConsoleEventDataSchema, + }), + z.object({ + type: z.literal("progress"), + data: sseProgressEventDataSchema, + }), + z.object({ + type: z.literal("start"), + data: z.string(), + }), + z.object({ + type: z.literal("done"), + data: z.string(), + }), +]); /** * A Braintrust stream. This is a wrapper around a ReadableStream of `BraintrustStreamChunk`, diff --git a/js/src/graph-framework.ts b/js/src/graph-framework.ts deleted file mode 100644 index 5d6766da6..000000000 --- a/js/src/graph-framework.ts +++ /dev/null @@ -1,499 +0,0 @@ -import { newId, Prompt } from "./logger"; -import type { - FunctionIdType as FunctionId, - GraphDataType as GraphData, - GraphNodeType as GraphNode, - GraphEdgeType as GraphEdge, - PromptBlockDataType as PromptBlockData, -} from "./generated_plain_types"; - -export interface BuildContext { - getFunctionId(functionObj: unknown): Promise; -} - -// Base interface for all node types -export interface Node { - readonly id: string; - __type: "node"; - build(context: BuildContext): Promise; - addDependency(dependency: Dependency): void; -} - -// type CallArgs = ProxyVariable | Node | Record; - -export type NodeLike = Node | Prompt | ProxyVariable; - -export type LazyGraphNode = { - type: "lazy"; - id: string; -}; - -// Graph builder class to convert functional chains to GraphData -export class GraphBuilder { - private nodes = new Map(); - private edges: Record = {}; - - private nodeLikeNodes = new Map(); // Maps node-like objects, like prompts, to their nodes - - // Special nodes - public readonly IN: InputNode; - public readonly OUT: OutputNode; - - constructor() { - // Create input and output nodes - this.IN = this.createInputNode(); - this.OUT = this.createOutputNode(); - } - - // Create the final GraphData object - public async build(context: BuildContext): Promise { - const nodes = await Promise.all( - Array.from(this.nodes.values()).map(async (node) => [ - node.id, - await node.build(context), - ]), - ); - - return { - type: "graph", - nodes: Object.fromEntries(nodes), // XXX Need to resolve the lazy nodes - edges: this.edges, - }; - } - - public addEdge({ - source, - sourceVar, - target, - targetVar, - expr, - purpose, - }: { - source: NodeLike; - sourceVar?: string; - target: NodeLike; - targetVar?: string; - expr?: string; - purpose: GraphEdge["purpose"]; - }) { - const [sourceNode, sourcePath] = this.resolveNode(source); - if (sourcePath.length > 0) { - // XXX Maybe we can remove these paths? - throw new Error("Source path must be empty"); - } - const [targetNode, targetPath] = this.resolveNode(target); - if (targetPath.length > 0) { - throw new Error("Target path must be empty"); - } - const id = this.generateId(); - - sourceVar = sourceVar ?? "output"; - targetVar = - targetVar ?? (purpose === "data" ? "input" : this.generateId("control")); - - // Make sure this variable name doesn't already exist as a target variable - for (const edge of Object.values(this.edges)) { - if ( - edge.target.node === targetNode.id && - edge.target.variable === targetVar - ) { - throw new Error( - `Variable name ${targetVar} already set on ${targetNode.id}`, - ); - } - } - - this.edges[id] = { - source: { node: sourceNode.id, variable: sourceVar }, - target: { node: targetNode.id, variable: targetVar }, - purpose, - }; - } - - public resolveNode(node: NodeLike): [Node, string[]] { - if (node instanceof Prompt) { - const cached = this.nodeLikeNodes.get(node); - if (cached) { - return [cached, []]; - } - const promptNode = this.createPromptNode(node); - this.nodeLikeNodes.set(node, promptNode); - return [promptNode, []]; - } else if (isProxyVariable(node)) { - return proxyVariableToNode(node); - } else { - return [node, []]; - } - } - - // Create a literal node - public literal(value: T): LiteralNode { - const preview = ( - typeof value === "string" ? value : JSON.stringify(value) - ).slice(0, 16); - const id = this.generateId(`literal-${preview}`); - const literalNode = new LiteralNode(this, id, value); - this.nodes.set(id, literalNode); - return literalNode; - } - - public gate(options: { condition: string }): GateNode { - const id = this.generateId("gate"); - const gateNode = new GateNode(this, id, options.condition); - this.nodes.set(id, gateNode); - return gateNode; - } - - public aggregator(): AggregatorNode { - const id = this.generateId("aggregator"); - const aggregatorNode = new AggregatorNode(this, id); - this.nodes.set(id, aggregatorNode); - return aggregatorNode; - } - - public promptTemplate(options: { - prompt: PromptBlockData; - }): PromptTemplateNode { - const id = this.generateId("prompt-template"); - const promptTemplateNode = new PromptTemplateNode(this, id, options.prompt); - this.nodes.set(id, promptTemplateNode); - return promptTemplateNode; - } - - // public call(node: NodeLike, input: CallArgs): Node { - // const [resolvedNode, path] = this.resolveNode(node); - // if (resolvedNode instanceof SingleInputNode) { - // return resolvedNode.call(input, path); - // } else { - // throw new Error("Node must be a SingleInputNode"); - // } - // } - - // Helper to generate node IDs - private generateId(name?: string): string { - const uuid = newId(); - if (name) { - return `${name}-${uuid.slice(0, 8)}`; - } else { - return uuid; - } - } - - // Create an input node - private createInputNode(): InputNode { - const id = this.generateId("input"); - const inputNode = new InputNode(this, id); - this.nodes.set(id, inputNode); - return inputNode; - } - - // Create an output node - private createOutputNode(): OutputNode { - const id = this.generateId("output"); - const outputNode = new OutputNode(this, id); - this.nodes.set(id, outputNode); - return outputNode; - } - - // Create a prompt node from a CodePrompt - private createPromptNode(prompt: Prompt): PromptNode { - const id = this.generateId(`prompt-${prompt.slug}`); - - const promptNode = new PromptNode(this, id, prompt); - this.nodes.set(id, promptNode); - return promptNode; - } -} - -export type ProxyVariable = { - [key: string]: ProxyVariable; -}; - -function isProxyVariable(node: unknown): node is ProxyVariable { - return ( - typeof node === "object" && - node !== null && - "__type" in node && - // @ts-ignore - node.__type === "proxy-variable" - ); -} - -function proxyVariableToNode(proxy: ProxyVariable): [Node, string[]] { - // @ts-ignore - return [proxy.__node, proxy.__path]; -} - -// // Create a proxy handler that captures property access paths -// function createVariableProxy({ -// path, -// node, -// }: { -// path: string[]; -// node: Node; -// }): ProxyVariable { -// // eslint-disable-next-line @typescript-eslint/consistent-type-assertions -// return new Proxy({} as ProxyVariable, { -// get(target, prop) { -// if (typeof prop === "string") { -// if (prop === "__type") { -// return "proxy-variable"; -// } else if (prop === "__node") { -// return node; -// } else if (prop === "__path") { -// return path; -// } - -// const newPath = [...path, prop]; - -// // Return a variable reference for terminal properties -// // or a new proxy for further chaining -// return createVariableProxy({ path: newPath, node }); -// } -// return undefined; -// }, -// has(target, prop) { -// return typeof prop === "string"; -// }, -// }); -// } - -// Type for transform functions -export type TransformFn = (input: ProxyVariable) => Node; - -interface Dependency { - node: Node; - sourceVar?: string; - targetVar?: string; - expr?: string; -} - -// Base Node class for common functionality -abstract class BaseNode implements Node { - public readonly __type = "node"; - public dependencies: Dependency[] = []; - - constructor( - protected graph: GraphBuilder, - public readonly id: string, - ) {} - - public addDependency(dependency: Dependency) { - this.dependencies.push(dependency); - } - - abstract build(context: BuildContext): Promise; -} - -// abstract class SingleOutputNode extends BaseNode { -// // Connect this node to another node -// public then(...args: Array): Node { -// // eslint-disable-next-line @typescript-eslint/consistent-type-assertions -// const callableThis = this as unknown as Node; -// let lastNode: Node = callableThis; - -// // Connect each arg to this node -// for (const arg of args) { -// // Handle different types of arguments -// if (typeof arg === "function") { -// // This function is expected to take dependencies as needed on the argsProxy. -// const argsProxy = createVariableProxy({ path: [], node: callableThis }); -// const result = arg(argsProxy); -// lastNode = result; -// } else { -// const [node, path] = this.graph.resolveNode(arg); -// lastNode = node; -// node.addDependency({ node: callableThis, expr: escapePath(path) }); -// } -// } - -// return lastNode; -// } -// } - -// abstract class SingleInputNode extends BaseNode { -// public call(input: CallArgs, path?: string[]): Node { -// if (isProxyVariable(input)) { -// const [sourceNode, sourcePath] = proxyVariableToNode(input); -// this.addDependency({ node: sourceNode, expr: escapePath(sourcePath) }); -// } else if (isNode(input)) { -// this.addDependency({ -// node: input, -// expr: path ? escapePath(path) : undefined, -// }); -// } else { -// for (const [targetVar, source] of Object.entries(input)) { -// const [sourceNode, sourcePath] = this.graph.resolveNode(source); -// this.addDependency({ -// node: sourceNode, -// expr: sourcePath ? escapePath(sourcePath) : undefined, -// targetVar, -// }); -// } -// } -// return this; -// } -// } - -// function isNode(node: unknown): node is Node { -// return ( -// typeof node === "object" && -// node !== null && -// "__type" in node && -// // @ts-ignore -// node.__type === "node" -// ); -// } - -// Input node (entry point to the graph) -export class InputNode extends BaseNode implements Node { - constructor(graph: GraphBuilder, id: string) { - super(graph, id); - } - - public async build(context: BuildContext): Promise { - return { - type: "input", - description: "Input to the graph", - }; - } -} - -// Output node (exit point from the graph) -export class OutputNode extends BaseNode implements Node { - constructor(graph: GraphBuilder, id: string) { - super(graph, id); - } - - public async build(context: BuildContext): Promise { - return { - type: "output", - description: "Output of the graph", - }; - } -} - -// Prompt node (wrapper for CodePrompt) -export class PromptNode extends BaseNode implements Node { - constructor( - graph: GraphBuilder, - id: string, - private prompt: Prompt, - ) { - super(graph, id); - } - - public async build(context: BuildContext): Promise { - return { - type: "function", - function: await context.getFunctionId(this.prompt), - }; - } -} - -// Gate node for conditional branching -export class GateNode extends BaseNode implements Node { - constructor( - graph: GraphBuilder, - id: string, - private condition: string, - ) { - super(graph, id); - } - - public async build(context: BuildContext): Promise { - return { - type: "gate", - description: "Conditional gate", - condition: this.condition, - }; - } -} - -export class AggregatorNode extends BaseNode implements Node { - constructor(graph: GraphBuilder, id: string) { - super(graph, id); - } - - public async build(context: BuildContext): Promise { - return { - type: "aggregator", - description: "Aggregator", - }; - } -} - -export class PromptTemplateNode extends BaseNode implements Node { - constructor( - graph: GraphBuilder, - id: string, - private prompt: PromptBlockData, - ) { - super(graph, id); - } - - public async build(context: BuildContext): Promise { - return { - type: "prompt_template", - prompt: this.prompt, - }; - } -} - -// Literal node for constant values -export class LiteralNode extends BaseNode implements Node { - constructor( - graph: GraphBuilder, - id: string, - private value: T, - ) { - super(graph, id); - } - - public async build(context: BuildContext): Promise { - return { - type: "literal", - value: this.value, - }; - } -} - -// Create a graph instance with IN and OUT nodes -export function createGraph(): GraphBuilder { - const graphBuilder = new GraphBuilder(); - return graphBuilder; -} - -// XXX write tests -export function escapePath(parts: string[]): string | undefined { - if (parts.length === 0) { - return undefined; - } - return parts - .map((part) => { - if (/[^\w-]/.test(part)) { - // Escape special characters properly - const escaped = part.replace(/["\\]/g, "\\$&"); - return `"${escaped}"`; - } - return part; - }) - .join("."); -} - -export function unescapePath(path: string): string[] { - const regex = /"((?:\\["\\]|[^"\\])*)"|([^\.]+)/g; - const matches = path.match(regex); - return matches - ? matches.map((match) => { - if (match.startsWith('"')) { - // Remove surrounding quotes and unescape special characters - return match.slice(1, -1).replace(/\\(["\\])/g, "$1"); - } - return match; - }) - : []; -} - -// Export the graph constructor -export default { createGraph }; diff --git a/js/src/id-gen.ts b/js/src/id-gen.ts index 67925916a..d95e1a0bc 100644 --- a/js/src/id-gen.ts +++ b/js/src/id-gen.ts @@ -136,15 +136,10 @@ export function resolveUseLegacyUuidIds(): boolean { * This eliminates global state and makes tests parallelizable. * Each caller gets their own generator instance. * - * Honors an explicitly-installed `globalThis.BRAINTRUST_ID_GENERATOR` (e.g. set - * by `@braintrust/otel`'s `setupOtelCompat()`). Otherwise it defaults to - * OpenTelemetry-compatible hex IDs, falling back to legacy UUID IDs when - * BRAINTRUST_LEGACY_IDS is set. + * Defaults to OpenTelemetry-compatible hex IDs, falling back to legacy UUID + * IDs when BRAINTRUST_LEGACY_IDS is set. */ export function getIdGenerator(): IDGenerator { - if (globalThis.BRAINTRUST_ID_GENERATOR !== undefined) { - return new globalThis.BRAINTRUST_ID_GENERATOR(); - } return resolveUseLegacyUuidIds() ? new UUIDGenerator() : new OTELIDGenerator(); diff --git a/js/src/instrumentation/config.ts b/js/src/instrumentation/config.ts index e8df49037..edcbe083e 100644 --- a/js/src/instrumentation/config.ts +++ b/js/src/instrumentation/config.ts @@ -1,16 +1,13 @@ export interface InstrumentationIntegrationsConfig { openai?: boolean; anthropic?: boolean; - vercel?: boolean; aisdk?: boolean; - google?: boolean; googleGenAI?: boolean; googleADK?: boolean; huggingface?: boolean; claudeAgentSDK?: boolean; cloudflareAIChat?: boolean; cloudflareThink?: boolean; - cursor?: boolean; cursorSDK?: boolean; mastra?: boolean; openAIAgents?: boolean; @@ -20,8 +17,6 @@ export interface InstrumentationIntegrationsConfig { ollama?: boolean; cohere?: boolean; groq?: boolean; - bedrock?: boolean; - awsBedrock?: boolean; awsBedrockRuntime?: boolean; genkit?: boolean; gitHubCopilot?: boolean; @@ -69,14 +64,14 @@ const envIntegrationAliases: Record< aisdk: "aisdk", "ai-sdk": "aisdk", "vercel-ai": "aisdk", - vercel: "vercel", + vercel: "aisdk", claudeagentsdk: "claudeAgentSDK", "claude-agent-sdk": "claudeAgentSDK", cloudflareaichat: "cloudflareAIChat", "cloudflare-ai-chat": "cloudflareAIChat", "@cloudflare/ai-chat": "cloudflareAIChat", cloudflarethink: "cloudflareThink", - cursor: "cursor", + cursor: "cursorSDK", "cursor-sdk": "cursorSDK", cursorsdk: "cursorSDK", mastra: "mastra", @@ -84,7 +79,7 @@ const envIntegrationAliases: Record< openaiagents: "openAIAgents", "openai-agents-core": "openAIAgents", openaiagentscore: "openAIAgents", - google: "google", + google: "googleGenAI", "google-genai": "googleGenAI", googlegenai: "googleGenAI", huggingface: "huggingface", @@ -100,9 +95,9 @@ const envIntegrationAliases: Record< cohere: "cohere", groq: "groq", "groq-sdk": "groq", - bedrock: "bedrock", - "aws-bedrock": "awsBedrock", - awsbedrock: "awsBedrock", + bedrock: "awsBedrockRuntime", + "aws-bedrock": "awsBedrockRuntime", + awsbedrock: "awsBedrockRuntime", "aws-bedrock-runtime": "awsBedrockRuntime", awsbedrockruntime: "awsBedrockRuntime", "@aws-sdk/client-bedrock-runtime": "awsBedrockRuntime", @@ -129,16 +124,13 @@ export function getDefaultInstrumentationIntegrations(): Record< openai: true, openaiCodexSDK: true, anthropic: true, - vercel: true, aisdk: true, - google: true, googleGenAI: true, googleADK: true, huggingface: true, claudeAgentSDK: true, cloudflareAIChat: true, cloudflareThink: true, - cursor: true, cursorSDK: true, mastra: true, openAIAgents: true, @@ -148,8 +140,6 @@ export function getDefaultInstrumentationIntegrations(): Record< ollama: true, cohere: true, groq: true, - bedrock: true, - awsBedrock: true, awsBedrockRuntime: true, genkit: true, gitHubCopilot: true, diff --git a/js/src/instrumentation/core/channel-tracing.ts b/js/src/instrumentation/core/channel-tracing.ts index 1f3ff47d0..7d69fb12b 100644 --- a/js/src/instrumentation/core/channel-tracing.ts +++ b/js/src/instrumentation/core/channel-tracing.ts @@ -1,11 +1,7 @@ import { debugLogger } from "../../debug-logger"; import type { IsoChannelHandlers, IsoTracingChannel } from "../../isomorph"; -import { - _internalGetGlobalState, - BRAINTRUST_CURRENT_SPAN_STORE, - startSpan, -} from "../../logger"; -import type { CurrentSpanStore, Span } from "../../logger"; +import { _internalGetGlobalState, startSpan } from "../../logger"; +import type { Span } from "../../logger"; import { withSpanInstrumentationName, type SpanInstrumentationName, @@ -316,13 +312,7 @@ function bindCurrentSpanStoreToStart< const state = _internalGetGlobalState(); const startChannel = tracingChannel.start; const contextManager = state?.contextManager; - const currentSpanStore = contextManager - ? ( - contextManager as { - [BRAINTRUST_CURRENT_SPAN_STORE]?: CurrentSpanStore; - } - )[BRAINTRUST_CURRENT_SPAN_STORE] - : undefined; + const currentSpanStore = contextManager?.getCurrentSpanStore(); if (!currentSpanStore || !startChannel) { return; diff --git a/js/src/instrumentation/index.test.ts b/js/src/instrumentation/index.test.ts index fd5442745..b51519344 100644 --- a/js/src/instrumentation/index.test.ts +++ b/js/src/instrumentation/index.test.ts @@ -8,6 +8,7 @@ describe("instrumentation public API", () => { "braintrustEveInstrumentation", "braintrustFlueInstrumentation", "configureInstrumentation", + "registerOtelFlush", ]); }); }); diff --git a/js/src/instrumentation/index.ts b/js/src/instrumentation/index.ts index e148f2dc5..67853cce8 100644 --- a/js/src/instrumentation/index.ts +++ b/js/src/instrumentation/index.ts @@ -15,3 +15,6 @@ export { braintrustEveInstrumentation } from "./providers/eve-instrumentation"; // Configuration API export { configureInstrumentation } from "./registry"; export type { InstrumentationConfig } from "./registry"; + +// Integration hook used by @braintrust/otel so core flushes also drain OTEL. +export { registerOtelFlush } from "../logger"; diff --git a/js/src/instrumentation/instrumentation-consumers.test.ts b/js/src/instrumentation/instrumentation-consumers.test.ts index ce4837bb5..9499bff31 100644 --- a/js/src/instrumentation/instrumentation-consumers.test.ts +++ b/js/src/instrumentation/instrumentation-consumers.test.ts @@ -128,7 +128,7 @@ const integrationCases: Array< > = [ ["aisdk", [registrations.aiSDK]], ["anthropic", [registrations.anthropic]], - ["bedrock", [registrations.bedrockRuntime]], + ["awsBedrockRuntime", [registrations.bedrockRuntime]], ["claudeAgentSDK", [registrations.claudeAgentSDK]], ["cloudflareAgents", [registrations.cloudflareAgents]], ["cloudflareAIChat", [registrations.cloudflareAIChat]], @@ -187,15 +187,8 @@ describe("registerInstrumentationConsumers", () => { }, ); - it.each([ - ["vercel", registrations.aiSDK], - ["google", registrations.googleGenAI], - ["cursor", registrations.cursorSDK], - ["awsBedrock", registrations.bedrockRuntime], - ["awsBedrockRuntime", registrations.bedrockRuntime], - ["langgraph", registrations.langChain], - ] as const)( - "honors the legacy or related %s disable flag", + it.each([["langgraph", registrations.langChain]] as const)( + "honors the related %s disable flag", (integration, register) => { registerInstrumentationConsumers({ integrations: { [integration]: false }, diff --git a/js/src/instrumentation/instrumentation-consumers.ts b/js/src/instrumentation/instrumentation-consumers.ts index ed32522a9..57d01b43d 100644 --- a/js/src/instrumentation/instrumentation-consumers.ts +++ b/js/src/instrumentation/instrumentation-consumers.ts @@ -46,8 +46,7 @@ export function registerInstrumentationConsumers( registerAnthropicInstrumentation(); } - // Support both 'aisdk' and legacy 'vercel' config keys - if (integrations.aisdk !== false && integrations.vercel !== false) { + if (integrations.aisdk !== false) { registerAISDKInstrumentation(); } @@ -59,7 +58,7 @@ export function registerInstrumentationConsumers( registerCloudflareThinkInstrumentation(); } - if (integrations.cursorSDK !== false && integrations.cursor !== false) { + if (integrations.cursorSDK !== false) { registerCursorSDKInstrumentation(); } @@ -67,8 +66,7 @@ export function registerInstrumentationConsumers( registerOpenAIAgentsInstrumentation(); } - // Support both 'googleGenAI' and legacy 'google' config keys - if (integrations.googleGenAI !== false && integrations.google !== false) { + if (integrations.googleGenAI !== false) { registerGoogleGenAIInstrumentation(); } @@ -109,11 +107,7 @@ export function registerInstrumentationConsumers( registerGroqInstrumentation(); } - if ( - integrations.bedrock !== false && - integrations.awsBedrock !== false && - integrations.awsBedrockRuntime !== false - ) { + if (integrations.awsBedrockRuntime !== false) { registerBedrockRuntimeInstrumentation(); } diff --git a/js/src/instrumentation/providers/claude-agent-sdk-instrumentation.test.ts b/js/src/instrumentation/providers/claude-agent-sdk-instrumentation.test.ts index 93265e4a3..1394ac919 100644 --- a/js/src/instrumentation/providers/claude-agent-sdk-instrumentation.test.ts +++ b/js/src/instrumentation/providers/claude-agent-sdk-instrumentation.test.ts @@ -32,18 +32,22 @@ vi.mock("../core/stream-patcher", () => ({ import { registerClaudeAgentSDKInstrumentation } from "./claude-agent-sdk-instrumentation"; import iso from "../../isomorph"; -import { startSpan } from "../../logger"; +import { _internalStartSpan as startSpan } from "../../logger"; const mockNewTracingChannel = iso.newTracingChannel as ReturnType; // Mock the logger module -vi.mock("../../logger", () => ({ - startSpan: vi.fn(() => ({ +vi.mock("../../logger", () => { + const startSpan = vi.fn(() => ({ log: vi.fn(), end: vi.fn(), export: vi.fn(() => Promise.resolve({})), - })), -})); + })); + return { + startSpan, + _internalStartSpan: startSpan, + }; +}); // Mock utility modules vi.mock("../../../util/index", () => ({ diff --git a/js/src/instrumentation/providers/claude-agent-sdk-instrumentation.ts b/js/src/instrumentation/providers/claude-agent-sdk-instrumentation.ts index 9944ebe60..32b5ab345 100644 --- a/js/src/instrumentation/providers/claude-agent-sdk-instrumentation.ts +++ b/js/src/instrumentation/providers/claude-agent-sdk-instrumentation.ts @@ -1,7 +1,7 @@ import type { ChannelMessage } from "../core/channel-definitions"; import { isAsyncIterable, patchStreamIfNeeded } from "../core/stream-patcher"; import type { IsoChannelHandlers } from "../../isomorph"; -import { startSpan as startBaseSpan } from "../../logger"; +import { _internalStartSpan as startBaseSpan } from "../../logger"; import type { Span } from "../../logger"; import { INSTRUMENTATION_NAMES, diff --git a/js/src/instrumentation/providers/claude-agent-sdk-local-tool-spans.ts b/js/src/instrumentation/providers/claude-agent-sdk-local-tool-spans.ts index 1a1c7db88..2ba99e8c9 100644 --- a/js/src/instrumentation/providers/claude-agent-sdk-local-tool-spans.ts +++ b/js/src/instrumentation/providers/claude-agent-sdk-local-tool-spans.ts @@ -1,4 +1,4 @@ -import { startSpan as startBaseSpan, withCurrent } from "../../logger"; +import { _internalStartSpan as startBaseSpan, withCurrent } from "../../logger"; import { INSTRUMENTATION_NAMES, withSpanInstrumentationName, diff --git a/js/src/instrumentation/providers/cloudflare-ai-chat-consumer.test.ts b/js/src/instrumentation/providers/cloudflare-ai-chat-consumer.test.ts index 860ab0b18..f0a0c3a72 100644 --- a/js/src/instrumentation/providers/cloudflare-ai-chat-consumer.test.ts +++ b/js/src/instrumentation/providers/cloudflare-ai-chat-consumer.test.ts @@ -1,25 +1,19 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -const { - mockInternalGetGlobalState, - mockStartSpan, - mockWithCurrent, - mockCurrentSpanStoreSymbol, -} = vi.hoisted(() => ({ - mockCurrentSpanStoreSymbol: Symbol.for("braintrust.currentSpanStore"), - mockInternalGetGlobalState: vi.fn(() => undefined), - mockStartSpan: vi.fn(), - mockWithCurrent: vi.fn((_span: unknown, callback: () => unknown) => - callback(), - ), -})); +const { mockInternalGetGlobalState, mockStartSpan, mockWithCurrent } = + vi.hoisted(() => ({ + mockInternalGetGlobalState: vi.fn(() => undefined), + mockStartSpan: vi.fn(), + mockWithCurrent: vi.fn((_span: unknown, callback: () => unknown) => + callback(), + ), + })); vi.mock("../../isomorph", () => ({ default: { newTracingChannel: vi.fn() }, })); vi.mock("../../logger", () => ({ - BRAINTRUST_CURRENT_SPAN_STORE: mockCurrentSpanStoreSymbol, _internalGetGlobalState: () => mockInternalGetGlobalState(), startSpan: (...args: unknown[]) => (mockStartSpan as any)(...args), withCurrent: (...args: unknown[]) => (mockWithCurrent as any)(...args), diff --git a/js/src/instrumentation/providers/cloudflare-ai-chat-consumer.ts b/js/src/instrumentation/providers/cloudflare-ai-chat-consumer.ts index a75ad3ea6..2258ea666 100644 --- a/js/src/instrumentation/providers/cloudflare-ai-chat-consumer.ts +++ b/js/src/instrumentation/providers/cloudflare-ai-chat-consumer.ts @@ -1,12 +1,11 @@ import type { ChannelMessage } from "../core/channel-definitions"; import type { IsoChannelHandlers, IsoTracingChannel } from "../../isomorph"; import { - BRAINTRUST_CURRENT_SPAN_STORE, _internalGetGlobalState, startSpan as startBaseSpan, withCurrent, } from "../../logger"; -import type { CurrentSpanStore, Span } from "../../logger"; +import type { Span } from "../../logger"; import { debugLogger } from "../../debug-logger"; import { INSTRUMENTATION_NAMES, @@ -134,13 +133,7 @@ class CloudflareAIChatInstrumentationConsumer { const globalState = _internalGetGlobalState(); const contextManager = globalState?.contextManager; const startChannel = tracingChannel.start; - const currentSpanStore = contextManager - ? ( - contextManager as { - [BRAINTRUST_CURRENT_SPAN_STORE]?: CurrentSpanStore; - } - )[BRAINTRUST_CURRENT_SPAN_STORE] - : undefined; + const currentSpanStore = contextManager?.getCurrentSpanStore(); if (!startChannel || !currentSpanStore || !contextManager) { return; diff --git a/js/src/instrumentation/providers/cloudflare-think-instrumentation.ts b/js/src/instrumentation/providers/cloudflare-think-instrumentation.ts index 66196becf..17751542a 100644 --- a/js/src/instrumentation/providers/cloudflare-think-instrumentation.ts +++ b/js/src/instrumentation/providers/cloudflare-think-instrumentation.ts @@ -1,11 +1,7 @@ import type { ChannelMessage } from "../core/channel-definitions"; import type { IsoChannelHandlers } from "../../isomorph"; -import { - BRAINTRUST_CURRENT_SPAN_STORE, - _internalGetGlobalState, - startSpan, -} from "../../logger"; -import type { CurrentSpanStore, Span } from "../../logger"; +import { _internalGetGlobalState, startSpan } from "../../logger"; +import type { Span } from "../../logger"; import { INSTRUMENTATION_NAMES, withSpanInstrumentationName, @@ -63,13 +59,7 @@ class CloudflareThinkInstrumentationConsumer { const states = new WeakMap(); const state = _internalGetGlobalState(); const contextManager = state?.contextManager; - const currentSpanStore = contextManager - ? ( - contextManager as { - [BRAINTRUST_CURRENT_SPAN_STORE]?: CurrentSpanStore; - } - )[BRAINTRUST_CURRENT_SPAN_STORE] - : undefined; + const currentSpanStore = contextManager?.getCurrentSpanStore(); const ensureState = ( event: ChannelMessage, diff --git a/js/src/instrumentation/providers/cursor-sdk-instrumentation.test.ts b/js/src/instrumentation/providers/cursor-sdk-instrumentation.test.ts index c30169c35..39d3582a6 100644 --- a/js/src/instrumentation/providers/cursor-sdk-instrumentation.test.ts +++ b/js/src/instrumentation/providers/cursor-sdk-instrumentation.test.ts @@ -12,6 +12,7 @@ vi.mock("../../isomorph", () => ({ vi.mock("../../logger", () => ({ startSpan: (...args: unknown[]) => mockStartSpan(...args), + _internalStartSpan: (...args: unknown[]) => mockStartSpan(...args), })); import iso from "../../isomorph"; diff --git a/js/src/instrumentation/providers/cursor-sdk-instrumentation.ts b/js/src/instrumentation/providers/cursor-sdk-instrumentation.ts index 41118be17..957d8a997 100644 --- a/js/src/instrumentation/providers/cursor-sdk-instrumentation.ts +++ b/js/src/instrumentation/providers/cursor-sdk-instrumentation.ts @@ -2,7 +2,7 @@ import { toLoggedError } from "../core"; import type { ChannelMessage } from "../core/channel-definitions"; import type { IsoChannelHandlers } from "../../isomorph"; import { debugLogger } from "../../debug-logger"; -import { startSpan as startBaseSpan } from "../../logger"; +import { _internalStartSpan as startBaseSpan } from "../../logger"; import type { Span } from "../../logger"; import { INSTRUMENTATION_NAMES, diff --git a/js/src/instrumentation/providers/eve-instrumentation.test.ts b/js/src/instrumentation/providers/eve-instrumentation.test.ts index 9e1a6bc28..48e6b5bff 100644 --- a/js/src/instrumentation/providers/eve-instrumentation.test.ts +++ b/js/src/instrumentation/providers/eve-instrumentation.test.ts @@ -93,9 +93,9 @@ describe("braintrustEveInstrumentation", () => { ]); }); - it("exports only the Eve instrumentation provider API", () => { - expect(braintrustExports.braintrustEveInstrumentation).toBe( - braintrustEveInstrumentation, + it("exports the Eve provider only from the instrumentation entrypoint", () => { + expect(braintrustExports).not.toHaveProperty( + "braintrustEveInstrumentation", ); expect(instrumentationExports.braintrustEveInstrumentation).toBe( braintrustEveInstrumentation, diff --git a/js/src/instrumentation/providers/flue-instrumentation.test.ts b/js/src/instrumentation/providers/flue-instrumentation.test.ts index ae6fbec8b..f1c05a12c 100644 --- a/js/src/instrumentation/providers/flue-instrumentation.test.ts +++ b/js/src/instrumentation/providers/flue-instrumentation.test.ts @@ -3,17 +3,14 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; const { mockDebugLog, mockCurrentParentSpan, - mockCurrentSpanStoreSymbol, mockCurrentSpanStore, mockFlush, mockStartSpan, } = vi.hoisted(() => { const currentParentSpan = { current: undefined as any }; - const currentSpanStoreSymbol = Symbol.for("braintrust.currentSpanStore"); return { mockDebugLog: vi.fn(), mockCurrentParentSpan: currentParentSpan, - mockCurrentSpanStoreSymbol: currentSpanStoreSymbol, mockCurrentSpanStore: { getStore: vi.fn(() => currentParentSpan.current), run: vi.fn((span: unknown, callback: () => unknown) => { @@ -38,12 +35,11 @@ vi.mock("../../debug-logger", () => ({ })); vi.mock("../../logger", () => ({ - BRAINTRUST_CURRENT_SPAN_STORE: mockCurrentSpanStoreSymbol, NOOP_SPAN: {}, flush: (...args: unknown[]) => mockFlush(...args), _internalGetGlobalState: () => ({ contextManager: { - [mockCurrentSpanStoreSymbol]: mockCurrentSpanStore, + getCurrentSpanStore: () => mockCurrentSpanStore, wrapSpanForStore: (span: unknown) => span, }, idGenerator: { @@ -53,6 +49,7 @@ vi.mock("../../logger", () => ({ }, }), startSpan: (...args: unknown[]) => mockStartSpan(...args), + _internalStartSpan: (...args: unknown[]) => mockStartSpan(...args), withCurrent: (span: unknown, callback: () => unknown) => { const previous = mockCurrentParentSpan.current; mockCurrentParentSpan.current = span; diff --git a/js/src/instrumentation/providers/flue-instrumentation.ts b/js/src/instrumentation/providers/flue-instrumentation.ts index 1426afde6..51fa32881 100644 --- a/js/src/instrumentation/providers/flue-instrumentation.ts +++ b/js/src/instrumentation/providers/flue-instrumentation.ts @@ -1,11 +1,10 @@ import { toLoggedError } from "../core"; import { debugLogger } from "../../debug-logger"; import { - BRAINTRUST_CURRENT_SPAN_STORE, NOOP_SPAN, flush, _internalGetGlobalState, - startSpan as startBaseSpan, + _internalStartSpan as startBaseSpan, withCurrent, } from "../../logger"; import type { Span, StartSpanArgs } from "../../logger"; @@ -1308,9 +1307,7 @@ function runWithCurrentSpanStore( ): Promise { const state = _internalGetGlobalState(); const contextManager = state?.contextManager; - const currentSpanStore = contextManager - ? Reflect.get(contextManager, BRAINTRUST_CURRENT_SPAN_STORE) - : undefined; + const currentSpanStore = contextManager?.getCurrentSpanStore(); if (contextManager && typeof currentSpanStore?.run === "function") { return currentSpanStore.run(contextManager.wrapSpanForStore(span), next); diff --git a/js/src/instrumentation/providers/genkit-instrumentation.ts b/js/src/instrumentation/providers/genkit-instrumentation.ts index e485295ea..16abe033c 100644 --- a/js/src/instrumentation/providers/genkit-instrumentation.ts +++ b/js/src/instrumentation/providers/genkit-instrumentation.ts @@ -8,7 +8,6 @@ import type { ChannelMessage } from "../core/channel-definitions"; import type { IsoChannelHandlers, IsoTracingChannel } from "../../isomorph"; import { _internalGetGlobalState, - BRAINTRUST_CURRENT_SPAN_STORE, startSpan as startBaseSpan, } from "../../logger"; import type { CurrentSpanStore, Span } from "../../logger"; @@ -291,13 +290,7 @@ function bindActionCurrentSpanStoreToStart< ) => void; } & object) | undefined; - const currentSpanStore = contextManager - ? ( - contextManager as { - [BRAINTRUST_CURRENT_SPAN_STORE]?: CurrentSpanStore; - } - )[BRAINTRUST_CURRENT_SPAN_STORE] - : undefined; + const currentSpanStore = contextManager?.getCurrentSpanStore(); if (!startChannel?.bindStore || !currentSpanStore) { return; diff --git a/js/src/instrumentation/providers/github-copilot-instrumentation.ts b/js/src/instrumentation/providers/github-copilot-instrumentation.ts index b5f14976f..bf53ebd72 100644 --- a/js/src/instrumentation/providers/github-copilot-instrumentation.ts +++ b/js/src/instrumentation/providers/github-copilot-instrumentation.ts @@ -1,5 +1,5 @@ import type { IsoChannelHandlers } from "../../isomorph"; -import { startSpan as startBaseSpan } from "../../logger"; +import { _internalStartSpan as startBaseSpan } from "../../logger"; import type { Span } from "../../logger"; import { INSTRUMENTATION_NAMES, diff --git a/js/src/instrumentation/providers/google-adk-instrumentation.test.ts b/js/src/instrumentation/providers/google-adk-instrumentation.test.ts index f4efcfc7d..12c8a9175 100644 --- a/js/src/instrumentation/providers/google-adk-instrumentation.test.ts +++ b/js/src/instrumentation/providers/google-adk-instrumentation.test.ts @@ -1,10 +1,6 @@ import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; -const { - mockCurrentSpanStoreSymbol: MOCK_CURRENT_SPAN_STORE_SYMBOL, - mockInternalGetGlobalState, -} = vi.hoisted(() => ({ - mockCurrentSpanStoreSymbol: Symbol.for("braintrust.currentSpanStore"), +const { mockInternalGetGlobalState } = vi.hoisted(() => ({ mockInternalGetGlobalState: vi.fn(() => undefined), })); @@ -31,7 +27,6 @@ vi.mock("../../logger", () => ({ startSpan: (...args: any[]) => (mockStartSpan as any)(...args), _internalGetGlobalState: (...args: any[]) => (mockInternalGetGlobalState as any)(...args), - BRAINTRUST_CURRENT_SPAN_STORE: MOCK_CURRENT_SPAN_STORE_SYMBOL, withCurrent: (_span: any, callback: () => unknown) => callback(), Attachment: class MockAttachment { reference: any; @@ -183,7 +178,7 @@ describe("registerGoogleADKInstrumentation", () => { const wrapSpanForStore = vi.fn(() => "wrapped-runner-store"); mockInternalGetGlobalState.mockReturnValue({ contextManager: { - [MOCK_CURRENT_SPAN_STORE_SYMBOL]: currentSpanStore, + getCurrentSpanStore: () => currentSpanStore, wrapSpanForStore, }, } as any); @@ -492,7 +487,7 @@ describe("registerGoogleADKInstrumentation", () => { const wrapSpanForStore = vi.fn(() => "wrapped-agent-store"); mockInternalGetGlobalState.mockReturnValue({ contextManager: { - [MOCK_CURRENT_SPAN_STORE_SYMBOL]: currentSpanStore, + getCurrentSpanStore: () => currentSpanStore, wrapSpanForStore, }, } as any); diff --git a/js/src/instrumentation/providers/google-adk-instrumentation.ts b/js/src/instrumentation/providers/google-adk-instrumentation.ts index 994779305..de631de88 100644 --- a/js/src/instrumentation/providers/google-adk-instrumentation.ts +++ b/js/src/instrumentation/providers/google-adk-instrumentation.ts @@ -2,7 +2,6 @@ import type { ChannelMessage } from "../core/channel-definitions"; import { isAsyncIterable, patchStreamIfNeeded } from "../core/stream-patcher"; import type { IsoChannelHandlers, IsoTracingChannel } from "../../isomorph"; import { - BRAINTRUST_CURRENT_SPAN_STORE, _internalGetGlobalState, startSpan as startBaseSpan, withCurrent, @@ -518,13 +517,7 @@ function bindCurrentSpanStoreToStart< ) => void; } & object) | undefined; - const currentSpanStore = contextManager - ? ( - contextManager as { - [BRAINTRUST_CURRENT_SPAN_STORE]?: CurrentSpanStore; - } - )[BRAINTRUST_CURRENT_SPAN_STORE] - : undefined; + const currentSpanStore = contextManager?.getCurrentSpanStore(); if (!startChannel?.bindStore || !currentSpanStore) { return; diff --git a/js/src/instrumentation/providers/google-genai-instrumentation.ts b/js/src/instrumentation/providers/google-genai-instrumentation.ts index 85e5ae049..bd3029147 100644 --- a/js/src/instrumentation/providers/google-genai-instrumentation.ts +++ b/js/src/instrumentation/providers/google-genai-instrumentation.ts @@ -8,7 +8,6 @@ import type { IsoChannelHandlers, IsoTracingChannel } from "../../isomorph"; import { _internalGetGlobalState, Attachment, - BRAINTRUST_CURRENT_SPAN_STORE, startSpan as startBaseSpan, type CurrentSpanStore, type Span, @@ -356,13 +355,7 @@ function bindCurrentSpanStoreToStart< ) => void; } & object) | undefined; - const currentSpanStore = contextManager - ? ( - contextManager as { - [BRAINTRUST_CURRENT_SPAN_STORE]?: CurrentSpanStore; - } - )[BRAINTRUST_CURRENT_SPAN_STORE] - : undefined; + const currentSpanStore = contextManager?.getCurrentSpanStore(); if (!startChannel?.bindStore || !currentSpanStore) { return; diff --git a/js/src/instrumentation/providers/openai-codex-instrumentation.test.ts b/js/src/instrumentation/providers/openai-codex-instrumentation.test.ts index f7250bb3b..c919b6d3b 100644 --- a/js/src/instrumentation/providers/openai-codex-instrumentation.test.ts +++ b/js/src/instrumentation/providers/openai-codex-instrumentation.test.ts @@ -12,6 +12,7 @@ vi.mock("../../isomorph", () => ({ vi.mock("../../logger", () => ({ startSpan: (...args: unknown[]) => mockStartSpan(...args), + _internalStartSpan: (...args: unknown[]) => mockStartSpan(...args), })); import iso from "../../isomorph"; diff --git a/js/src/instrumentation/providers/openai-codex-instrumentation.ts b/js/src/instrumentation/providers/openai-codex-instrumentation.ts index abf442120..eec4a36a2 100644 --- a/js/src/instrumentation/providers/openai-codex-instrumentation.ts +++ b/js/src/instrumentation/providers/openai-codex-instrumentation.ts @@ -2,8 +2,8 @@ import { toLoggedError } from "../core"; import type { ChannelMessage } from "../core/channel-definitions"; import type { IsoChannelHandlers } from "../../isomorph"; import { debugLogger } from "../../debug-logger"; -import { startSpan as startBaseSpan } from "../../logger"; -import type { Span, StartSpanArgs } from "../../logger"; +import { _internalStartSpan as startBaseSpan } from "../../logger"; +import type { Span } from "../../logger"; import { INSTRUMENTATION_NAMES, withSpanInstrumentationName, @@ -27,6 +27,8 @@ import type { OpenAICodexWebSearchItem, } from "../../vendor-sdk-types/openai-codex"; +type InternalStartSpanArgs = NonNullable[0]>; + type CodexRunState = { activeLlmSpan?: CodexLlmSpanState; activeItemSpans: Map; @@ -565,7 +567,7 @@ async function itemSpanArgs( item: OpenAICodexThreadItem, ): Promise< | { - start: StartSpanArgs; + start: InternalStartSpanArgs; end: Parameters[0]; } | undefined diff --git a/js/src/instrumentation/providers/pi-coding-agent-instrumentation.test.ts b/js/src/instrumentation/providers/pi-coding-agent-instrumentation.test.ts index 7e7d406ed..48b38ea1f 100644 --- a/js/src/instrumentation/providers/pi-coding-agent-instrumentation.test.ts +++ b/js/src/instrumentation/providers/pi-coding-agent-instrumentation.test.ts @@ -27,6 +27,7 @@ vi.mock("../../isomorph", async (importOriginal) => { vi.mock("../../logger", () => ({ startSpan: (...args: unknown[]) => mockStartSpan(...args), + _internalStartSpan: (...args: unknown[]) => mockStartSpan(...args), })); import { isAutoInstrumentationSuppressed } from "../auto-instrumentation-suppression"; diff --git a/js/src/instrumentation/providers/pi-coding-agent-instrumentation.ts b/js/src/instrumentation/providers/pi-coding-agent-instrumentation.ts index 11829e004..22537472b 100644 --- a/js/src/instrumentation/providers/pi-coding-agent-instrumentation.ts +++ b/js/src/instrumentation/providers/pi-coding-agent-instrumentation.ts @@ -2,7 +2,7 @@ import { toLoggedError } from "../core"; import type { ChannelMessage } from "../core/channel-definitions"; import iso, { type IsoAsyncLocalStorage } from "../../isomorph"; import { debugLogger } from "../../debug-logger"; -import { startSpan as startBaseSpan } from "../../logger"; +import { _internalStartSpan as startBaseSpan } from "../../logger"; import type { Span } from "../../logger"; import { INSTRUMENTATION_NAMES, diff --git a/js/src/instrumentation/registry.ts b/js/src/instrumentation/registry.ts index 010aee634..60520f46c 100644 --- a/js/src/instrumentation/registry.ts +++ b/js/src/instrumentation/registry.ts @@ -135,7 +135,7 @@ export const registry = new InstrumentationRegistry(); * * @example * ```typescript - * import { configureInstrumentation } from 'braintrust'; + * import { configureInstrumentation } from 'braintrust/instrumentation'; * * // Disable OpenAI instrumentation * configureInstrumentation({ diff --git a/js/src/logger-misc.test.ts b/js/src/logger-misc.test.ts index bd7051544..730bacb68 100644 --- a/js/src/logger-misc.test.ts +++ b/js/src/logger-misc.test.ts @@ -10,7 +10,7 @@ import { BaseAttachment, Attachment, ExternalAttachment, - initExperiment, + init, initLogger, NOOP_SPAN, permalink, @@ -368,7 +368,7 @@ describe("span.link", () => { expect(state.appUrl).toBeDefined(); // Create a test experiment - const experiment = initExperiment("test-experiment"); + const experiment = init({ project: "test-experiment" }); // Get a span within the experiment context const span = experiment.startSpan({ @@ -417,7 +417,7 @@ describe("span.link", () => { test("span.link handles missing experiment id", async () => { await _exportsForTestingOnly.simulateLoginForTests(); - const experiment = initExperiment("test-experiment"); + const experiment = init({ project: "test-experiment" }); const span = experiment.startSpan({ name: "test-span" }); span.end(); // Force parentObjectId to be undefined diff --git a/js/src/logger.test.ts b/js/src/logger.test.ts index 1dffdbb49..6b67268fd 100644 --- a/js/src/logger.test.ts +++ b/js/src/logger.test.ts @@ -11,9 +11,11 @@ import { } from "vitest"; import { _exportsForTestingOnly, + _internalStartSpan, init, initDataset, initLogger, + extractTraceContextFromHeaders, Prompt, BraintrustState, loadPrompt, @@ -898,7 +900,8 @@ test("legacy initDataset applies bt eval internal BTQL runtime value", async () }, }); - const dataset = initDataset("test-project", { + const dataset = initDataset({ + project: "test-project", dataset: "test-dataset", state, }); @@ -2997,7 +3000,7 @@ describe("parent precedence", () => { test("withParent + wrapTraced: child spans attach to current span (not directly to withParent)", async () => { const logger = initLogger({ projectName: "test", projectId: "pid" }); const outer = logger.startSpan({ name: "outer" }); - const parentStr = await outer.export(); + const parentContext = extractTraceContextFromHeaders(outer.inject())!; outer.end(); const inner = wrapTraced( @@ -3007,7 +3010,7 @@ describe("parent precedence", () => { { name: "inner" }, ); - await withParent(parentStr, () => inner()); + await withParent(parentContext, () => inner()); await memory.flush(); const events = await memory.drain(); @@ -3051,7 +3054,7 @@ describe("parent precedence", () => { const inner = wrapTraced( async function inner() { - startSpan({ name: "forced", parent: parentStr }).end(); + _internalStartSpan({ name: "forced", parent: parentStr }).end(); }, { name: "inner" }, ); @@ -3083,7 +3086,7 @@ describe("parent precedence", () => { const child = secondaryLogger.startSpan({ name: "child", - parent: parentStr, + parent: parentStr as never, }); child.end(); @@ -3114,7 +3117,7 @@ describe("parent precedence", () => { const child = secondaryExperiment.startSpan({ name: "child", - parent: parentStr, + parent: parentStr as never, }); child.end(); diff --git a/js/src/logger.ts b/js/src/logger.ts index 793cb96fc..62000ba17 100644 --- a/js/src/logger.ts +++ b/js/src/logger.ts @@ -37,8 +37,7 @@ import { batchItems, constructJsonArray, DatasetRecord, - DEFAULT_IS_LEGACY_DATASET, - ensureDatasetRecord, + ensureNewDatasetRecord, ExperimentEvent, ExperimentLogFullArgs, ExperimentLogPartialArgs, @@ -313,17 +312,20 @@ export type StartSpanArgs = { spanAttributes?: Record; startTime?: number; /** - * The parent to start this span under. May be an exported span slug string - * (from `span.export()`) or an opaque W3C trace-context (from - * {@link extractTraceContextFromHeaders}). + * An opaque W3C trace context returned by + * {@link extractTraceContextFromHeaders}. */ - parent?: string | PropagationContext; + parent?: PropagationContext; event?: StartSpanEventArgs; propagatedEvent?: StartSpanEventArgs; spanId?: string; parentSpanIds?: ParentSpanIds | MultiParentSpanIds; }; +type InternalStartSpanArgs = Omit & { + parent?: string | PropagationContext; +}; + export type EndSpanArgs = { endTime?: number; }; @@ -431,8 +433,8 @@ export interface Span extends Exportable { * parameters of {@link Span.startSpan} for usage details. * * Callers should treat the return value as opaque. The serialization format - * may change from time to time. If parsing is needed, use - * `SpanComponentsV3.fromStr`. + * may change from time to time. For cross-service propagation, prefer + * {@link Span.inject} and {@link extractTraceContextFromHeaders}. * * @returns Serialized representation of this span's identifiers. */ @@ -489,11 +491,6 @@ export interface Span extends Exportable { */ flush(): Promise; - /** - * Alias for `end`. - */ - close(args?: EndSpanArgs): number; - /** * Set the span's name, type, or other attributes after it's created. */ @@ -518,12 +515,8 @@ export interface Span extends Exportable { kind: "span"; } -export const BRAINTRUST_CURRENT_SPAN_STORE = Symbol.for( - "braintrust.currentSpanStore", -); - /** - * The type of AsyncLocalStorage exposed via {@link BRAINTRUST_CURRENT_SPAN_STORE}. + * The AsyncLocalStorage contract used by context-manager integrations. * * The stored value is intentionally opaque (`unknown`) because the concrete type * depends on the active context manager: @@ -540,6 +533,14 @@ export abstract class ContextManager { abstract runInContext(span: Span, callback: () => R): R; abstract getCurrentSpan(): Span | undefined; + /** + * Return the store used by instrumentation hooks to propagate the current + * span, when the context manager supports direct store binding. + */ + getCurrentSpanStore(): CurrentSpanStore | undefined { + return undefined; + } + /** * Returns the value to store in the ALS bound to a global hook's start event. * In default mode this is the Span itself; in OTEL mode it is the OTEL Context @@ -553,12 +554,14 @@ export abstract class ContextManager { class BraintrustContextManager extends ContextManager { private _currentSpan: IsoAsyncLocalStorage; - [BRAINTRUST_CURRENT_SPAN_STORE]: IsoAsyncLocalStorage; constructor() { super(); this._currentSpan = iso.newAsyncLocalStorage(); - this[BRAINTRUST_CURRENT_SPAN_STORE] = this._currentSpan; + } + + getCurrentSpanStore(): CurrentSpanStore { + return this._currentSpan; } getParentSpanIds(): ContextParentSpanIds | undefined { @@ -582,11 +585,8 @@ class BraintrustContextManager extends ContextManager { } } -// make sure to update @braintrust/otel package declare global { var BRAINTRUST_CONTEXT_MANAGER: (new () => ContextManager) | undefined; - var BRAINTRUST_ID_GENERATOR: (new () => IDGenerator) | undefined; - var BRAINTRUST_SPAN_COMPONENT: SpanComponent | undefined; } type SpanComponent = typeof SpanComponentsV3 | typeof SpanComponentsV4; @@ -597,11 +597,7 @@ type SpanComponent = typeof SpanComponentsV3 | typeof SpanComponentsV4; // serialize as V4, legacy UUID IDs serialize as V3. These must move together -- // serializing hex IDs via V3 would lose the compact encoding and risk // corrupting hex values that happen to parse as UUIDs. An explicit -// `globalThis.BRAINTRUST_SPAN_COMPONENT` (e.g. from `@braintrust/otel`) wins. function getSpanComponentsClass(): SpanComponent { - if (globalThis.BRAINTRUST_SPAN_COMPONENT) { - return globalThis.BRAINTRUST_SPAN_COMPONENT; - } return resolveUseLegacyUuidIds() ? SpanComponentsV3 : SpanComponentsV4; } @@ -611,6 +607,20 @@ export function getContextManager(): ContextManager { : new BraintrustContextManager(); } +/** + * Configure the context manager used by Braintrust. + * + * This extension point is intended for runtime integrations such as + * `@braintrust/otel` and `@braintrust/browser`. Passing `undefined` restores + * Braintrust's default context manager. + */ +export function configureContextManager( + contextManager: (new () => ContextManager) | undefined, +): void { + globalThis.BRAINTRUST_CONTEXT_MANAGER = contextManager; + _globalState?.[RESET_CONTEXT_MANAGER_STATE](); +} + /** * A fake implementation of the Span API which does nothing. This can be used as the default span. */ @@ -674,10 +684,6 @@ export class NoopSpan implements Span { public async flush(): Promise {} - public close(args?: EndSpanArgs): number { - return this.end(args); - } - public setAttributes(_args: Omit) {} public startSpanWithParents( @@ -3851,38 +3857,7 @@ type InitializedExperiment = */ export function init( options: Readonly>, -): InitializedExperiment; - -/** - * Legacy form of `init` which accepts the project name as the first parameter, - * separately from the remaining options. See `init(options)` for full details. - */ -export function init( - project: string, - options?: Readonly>, -): InitializedExperiment; - -/** - * Combined overload implementation of `init`. Do not call this directly. - * Instead, call `init(options)` or `init(project, options)`. - */ -export function init( - projectOrOptions: string | Readonly>, - optionalOptions?: Readonly>, ): InitializedExperiment { - const options = ((): Readonly> => { - if (typeof projectOrOptions === "string") { - return { ...optionalOptions, project: projectOrOptions }; - } else { - if (optionalOptions !== undefined) { - throw new Error( - "Cannot specify options struct as both parameters. Must call either init(project, options) or init(options).", - ); - } - return projectOrOptions; - } - })(); - const { project, experiment, @@ -4111,66 +4086,6 @@ export function init( return ret as InitializedExperiment; } -/** - * Alias for init(options). - */ -export function initExperiment( - options: Readonly>, -): InitializedExperiment; - -/** - * Alias for init(project, options). - */ -export function initExperiment( - project: string, - options?: Readonly>, -): InitializedExperiment; - -/** - * Combined overload implementation of `initExperiment`, which is an alias for - * `init`. Do not call this directly. Instead, call `initExperiment(options)` or - * `initExperiment(project, options)`. - */ -export function initExperiment( - projectOrOptions: string | Readonly>, - optionalOptions?: Readonly>, -): InitializedExperiment { - const options = ((): Readonly> => { - if (typeof projectOrOptions === "string") { - return { ...optionalOptions, project: projectOrOptions }; - } else { - if (optionalOptions !== undefined) { - throw new Error( - "Cannot specify options struct as both parameters. Must call either init(project, options) or init(options).", - ); - } - return projectOrOptions; - } - })(); - return init(options); -} - -/** - * @deprecated Use {@link init} instead. - */ -export function withExperiment( - project: string, - callback: (experiment: Experiment) => R, - options: Readonly & SetCurrentArg> = {}, -): R { - debugLogger - .forState(options.state) - .warn( - "withExperiment is deprecated and will be removed in a future version of braintrust. Simply create the experiment with `init`.", - ); - const experiment = init(project, options); - return callback(experiment); -} - -type UseOutputOption = { - useOutput?: IsLegacyDataset; -}; - declare global { // Set by the bt eval runner when CLI-controlled BTQL should be pushed down // into dataset-backed evals. @@ -4178,22 +4093,21 @@ declare global { var __bt_eval_internal_btql: Record | undefined; } -export type InitDatasetOptions = - FullLoginOptions & { - dataset?: string; - description?: string; - version?: string; - environment?: string; - snapshotName?: string; - projectId?: string; - metadata?: Record; - state?: BraintrustState; - _internal_btql?: Record; - } & UseOutputOption; +export type InitDatasetOptions = FullLoginOptions & { + dataset?: string; + description?: string; + version?: string; + environment?: string; + snapshotName?: string; + projectId?: string; + metadata?: Record; + state?: BraintrustState; + _internal_btql?: Record; +}; -export type FullInitDatasetOptions = { +export type FullInitDatasetOptions = { project?: string; -} & InitDatasetOptions; +} & InitDatasetOptions; async function getDatasetSnapshots( params: @@ -4416,52 +4330,11 @@ async function serializeDatasetForExperiment({ * @param options.orgName (Optional) The name of a specific organization to connect to. This is useful if you belong to multiple. * @param options.projectId The id of the project to create the dataset in. This takes precedence over `project` if specified. * @param options.metadata A dictionary with additional data about the dataset. The values in `metadata` can be any JSON-serializable type, but its keys must be strings. - * @param options.useOutput (Deprecated) If true, records will be fetched from this dataset in the legacy format, with the "expected" field renamed to "output". This option will be removed in a future version of Braintrust. * @returns The newly created Dataset. */ -export function initDataset< - IsLegacyDataset extends boolean = typeof DEFAULT_IS_LEGACY_DATASET, ->( - options: Readonly>, -): Dataset; - -/** - * Legacy form of `initDataset` which accepts the project name as the first - * parameter, separately from the remaining options. - * - * See `initDataset(options)` for full details. - */ -export function initDataset< - IsLegacyDataset extends boolean = typeof DEFAULT_IS_LEGACY_DATASET, ->( - project: string, - options?: Readonly>, -): Dataset; - -/** - * Combined overload implementation of `initDataset`. Do not call this - * directly. Instead, call `initDataset(options)` or `initDataset(project, - * options)`. - */ -export function initDataset< - IsLegacyDataset extends boolean = typeof DEFAULT_IS_LEGACY_DATASET, ->( - projectOrOptions: string | Readonly>, - optionalOptions?: Readonly>, -): Dataset { - const options = ((): Readonly> => { - if (typeof projectOrOptions === "string") { - return { ...optionalOptions, project: projectOrOptions }; - } else { - if (optionalOptions !== undefined) { - throw new Error( - "Cannot specify options struct as both parameters. Must call either initDataset(project, options) or initDataset(options).", - ); - } - return projectOrOptions; - } - })(); - +export function initDataset( + options: Readonly, +): Dataset { const { project, dataset, @@ -4476,7 +4349,6 @@ export function initDataset< forceLogin, projectId, metadata, - useOutput: legacy, state: stateArg, _internal_btql, } = options; @@ -4563,7 +4435,6 @@ export function initDataset< stateArg ?? _globalState, lazyMetadata, typeof resolvedVersion === "string" ? resolvedVersion : undefined, - legacy, internalBtql, resolvedVersion instanceof LazyValue || normalizedEnvironment !== undefined || @@ -4591,26 +4462,6 @@ export function initDataset< return datasetObject; } -/** - * @deprecated Use {@link initDataset} instead. - */ -export function withDataset< - R, - IsLegacyDataset extends boolean = typeof DEFAULT_IS_LEGACY_DATASET, ->( - project: string, - callback: (dataset: Dataset) => R, - options: Readonly> = {}, -): R { - debugLogger - .forState(options.state) - .warn( - "withDataset is deprecated and will be removed in a future version of braintrust. Simply create the dataset with `initDataset`.", - ); - const dataset = initDataset(project, options); - return callback(dataset); -} - // Note: the argument names *must* serialize the same way as the argument names // for the corresponding python function, because this function may be invoked // from arguments serialized elsewhere. @@ -6242,12 +6093,6 @@ export function wrapTraced any>( }, spanArgs)) as F; } -/** - * A synonym for `wrapTraced`. If you're porting from systems that use `traceable`, you can use this to - * make your codebase more consistent. - */ -export const traceable = wrapTraced; - /** * Lower-level alternative to `traced`. This allows you to start a span yourself, and can be useful in situations * where you cannot use callbacks. However, spans started with `startSpan` will not be marked as the "current span", @@ -6259,19 +6104,25 @@ export function startSpan(args?: StartSpanArgs & OptionalStateArg): Span { return startSpanImpl(args); } +export function _internalStartSpan( + args?: InternalStartSpanArgs & OptionalStateArg, +): Span { + return startSpanImpl(args); +} + /** @internal Start a span whose initial row is merged with concurrent writes. */ export function _internalStartSpanWithInitialMerge( - args?: StartSpanArgs & OptionalStateArg, + args?: InternalStartSpanArgs & OptionalStateArg, ): Span { return startSpanImpl({ ...args, [INITIAL_SPAN_WRITE_AS_MERGE]: true, - } as StartSpanArgs & OptionalStateArg & InitialSpanWriteAsMergeArg); + } as InternalStartSpanArgs & OptionalStateArg & InitialSpanWriteAsMergeArg); } /** @internal Start a span with SDK-controlled context fields. */ export function _internalStartSpanWithContext( - args: StartSpanArgs & OptionalStateArg, + args: InternalStartSpanArgs & OptionalStateArg, context: Record, ): Span { return startSpanImpl({ @@ -6299,7 +6150,7 @@ export function setFetch(fetch: typeof globalThis.fetch): void { } function startSpanImpl( - args?: StartSpanArgs & OptionalStateArg & InternalSpanContextArg, + args?: InternalStartSpanArgs & OptionalStateArg & InternalSpanContextArg, ): Span { const state = args?.state ?? _globalState; @@ -6346,7 +6197,10 @@ function startSpanImpl( propagatedState, }); } else { - return parentObject.startSpan(args); + // The internal entrypoint still accepts exported Braintrust parent slugs. + // Span implementations support that runtime form even though it is no + // longer part of their public method signature. + return parentObject.startSpan(args as StartSpanArgs); } } @@ -6433,6 +6287,14 @@ async function* asyncGeneratorWithCurrent( } export function withParent( + parent: PropagationContext, + callback: () => R, + state: BraintrustState | undefined = undefined, +): R { + return (state ?? _globalState).currentParent.run(parent, () => callback()); +} + +export function _internalWithParent( parent: string | PropagationContext, callback: () => R, state: BraintrustState | undefined = undefined, @@ -6536,22 +6398,11 @@ function validateAndSanitizeExperimentLogPartialArgs( } } - if ("input" in event && event.input && "inputs" in event && event.inputs) { - throw new Error( - "Only one of input or inputs (deprecated) can be specified. Prefer input.", - ); - } - if ("tags" in event && event.tags) { validateTags(event.tags); } - if ("inputs" in event) { - const { inputs, ...rest } = event; - return { input: inputs, ...rest }; - } else { - return { ...event }; - } + return { ...event }; } /** @@ -6762,18 +6613,9 @@ async function resolveAttachmentsToBase64>( // handling special fields like 'id'). function validateAndSanitizeExperimentLogFullArgs( event: ExperimentLogFullArgs, - hasDataset: boolean, ): ExperimentLogFullArgs { - if ( - ("input" in event && - !isEmpty(event.input) && - "inputs" in event && - !isEmpty(event.inputs)) || - (!("input" in event) && !("inputs" in event)) - ) { - throw new Error( - "Exactly one of input or inputs (deprecated) must be specified. Prefer input.", - ); + if (!("input" in event)) { + throw new Error("input must be specified"); } if (isEmpty(event.output)) { @@ -6783,14 +6625,6 @@ function validateAndSanitizeExperimentLogFullArgs( throw new Error("scores must be specified"); } - if (hasDataset && event.datasetRecordId === undefined) { - throw new Error("datasetRecordId must be specified when using a dataset"); - } else if (!hasDataset && event.datasetRecordId !== undefined) { - throw new Error( - "datasetRecordId cannot be specified when not using a dataset", - ); - } - return event; } @@ -7122,7 +6956,7 @@ export class Experiment ); } - event = validateAndSanitizeExperimentLogFullArgs(event, !!this.dataset); + event = validateAndSanitizeExperimentLogFullArgs(event); const span = this.startSpanImpl({ startTime: this.lastStartTime, event }); this.lastStartTime = span.end(); return span.id; @@ -7367,18 +7201,6 @@ export class Experiment async flush(): Promise { return await this.state.bgLogger().flush(); } - - /** - * @deprecated This function is deprecated. You can simply remove it from your code. - */ - public async close(): Promise { - debugLogger - .forState(this.state) - .warn( - "close is deprecated and will be removed in a future version of braintrust. It is now a no-op and can be removed", - ); - return this.id; - } } /** @@ -8002,10 +7824,6 @@ export class SpanImpl implements Span { return await this._state.bgLogger().flush(); } - public close(args?: EndSpanArgs): number { - return this.end(args); - } - public state(): BraintrustState { return this._state; } @@ -8086,9 +7904,7 @@ function splitLoggingData({ * * You should not create `Dataset` objects directly. Instead, use the `braintrust.initDataset()` method. */ -export class Dataset< - IsLegacyDataset extends boolean = typeof DEFAULT_IS_LEGACY_DATASET, -> extends ObjectFetcher> { +export class Dataset extends ObjectFetcher { private readonly lazyMetadata: LazyValue; private readonly __braintrust_dataset_marker = true; private newRecords = 0; @@ -8100,29 +7916,16 @@ export class Dataset< private state: BraintrustState, lazyMetadata: LazyValue, pinnedVersion?: string, - legacy?: IsLegacyDataset, _internal_btql?: Record, pinState?: DatasetPinState, ) { - // eslint-disable-next-line @typescript-eslint/consistent-type-assertions - const isLegacyDataset = (legacy ?? - DEFAULT_IS_LEGACY_DATASET) as IsLegacyDataset; - if (isLegacyDataset) { - debugLogger - .forState(state) - .warn( - `Records will be fetched from this dataset in the legacy format, with the "expected" field renamed to "output". Please update your code to use "expected", and use \`braintrust.initDataset()\` with \`{ useOutput: false }\`, which will become the default in a future version of Braintrust.`, - ); - } super( "dataset", pinnedVersion, (r: AnyDatasetRecord) => - // eslint-disable-next-line @typescript-eslint/consistent-type-assertions - ensureDatasetRecord( + ensureNewDatasetRecord( enrichAttachments(r, this.state), - isLegacyDataset, - ) as WithTransactionId>, + ) as WithTransactionId, _internal_btql, ); void this.__braintrust_dataset_marker; @@ -8213,13 +8016,9 @@ export class Dataset< private validateEvent({ metadata, - expected, - output, tags, }: { metadata?: Record; - expected?: unknown; - output?: unknown; tags?: string[]; }) { if (metadata !== undefined) { @@ -8230,12 +8029,6 @@ export class Dataset< } } - if (expected !== undefined && output !== undefined) { - throw new Error( - "Only one of expected or output (deprecated) can be specified. Prefer expected.", - ); - } - if (tags) { validateTags(tags); } @@ -8247,7 +8040,6 @@ export class Dataset< expected, metadata, tags, - output, origin, isMerge, }: { @@ -8256,18 +8048,15 @@ export class Dataset< expected?: unknown; metadata?: Record; tags?: string[]; - output?: unknown; origin?: ObjectReference; isMerge?: boolean; }): LazyValue { return new LazyValue(async () => { const dataset_id = await this.id; - const expectedValue = expected === undefined ? output : expected; - const args: BackgroundLogEvent = { id, input, - expected: expectedValue, + expected, tags, dataset_id, created: !isMerge ? new Date().toISOString() : undefined, //if we're merging/updating an event we will not add this ts @@ -8298,7 +8087,6 @@ export class Dataset< * JSON-serializable type, but its keys must be strings. * @param event.origin (Optional) a reference to the source object this dataset record was derived from. * @param event.id (Optional) a unique identifier for the event. If you don't provide one, Braintrust will generate one for you. - * @param event.output: (Deprecated) The output of your application. Use `expected` instead. * @returns The `id` of the logged record. */ public insert({ @@ -8307,7 +8095,6 @@ export class Dataset< metadata, tags, id, - output, origin, }: { readonly input?: unknown; @@ -8315,10 +8102,9 @@ export class Dataset< readonly tags?: string[]; readonly metadata?: Record; readonly id?: string; - readonly output?: unknown; readonly origin?: ObjectReference; }): string { - this.validateEvent({ metadata, expected, output, tags }); + this.validateEvent({ metadata, tags }); const rowId = id || uuidv4(); const args = this.createArgs( @@ -8328,7 +8114,6 @@ export class Dataset< expected, metadata, tags, - output, origin, isMerge: false, }), @@ -8365,7 +8150,7 @@ export class Dataset< readonly tags?: string[]; readonly metadata?: Record; }): string { - this.validateEvent({ metadata, expected, tags }); + this.validateEvent({ metadata, tags }); const args = this.createArgs( deepCopyEvent({ @@ -8561,18 +8346,6 @@ export class Dataset< return await this.state.bgLogger().flush(); } - /** - * @deprecated This function is deprecated. You can simply remove it from your code. - */ - public async close(): Promise { - debugLogger - .forState(this.state) - .warn( - "close is deprecated and will be removed in a future version of braintrust. It is now a no-op and can be removed", - ); - return this.id; - } - public static isDataset(data: unknown): data is Dataset { return ( typeof data === "object" && @@ -9337,7 +9110,7 @@ export class RemoteEvalParameters< } } -export type AnyDataset = Dataset; +export type AnyDataset = Dataset; /** * Summary of a score's performance. diff --git a/js/src/node/index.ts b/js/src/node/index.ts index 60e1efe09..e4a44adfe 100644 --- a/js/src/node/index.ts +++ b/js/src/node/index.ts @@ -56,4 +56,3 @@ import { configureNode } from "./config"; configureNode(); export * from "../exports"; -export * as default from "../exports"; diff --git a/js/src/openai-utils.ts b/js/src/openai-utils.ts index 5fa850915..9bc0cfa9c 100644 --- a/js/src/openai-utils.ts +++ b/js/src/openai-utils.ts @@ -5,8 +5,8 @@ import { isObject } from "../util/index"; // when OpenAI usage data does not include it. export const BRAINTRUST_CACHED_STREAM_METRIC = "__braintrust_cached_metric"; -export const LEGACY_CACHED_HEADER = "x-cached"; -export const X_CACHED_HEADER = "x-bt-cached"; +const LEGACY_CACHED_HEADER = "x-cached"; +const X_CACHED_HEADER = "x-bt-cached"; /** * Token name mappings for OpenAI metrics. @@ -63,7 +63,7 @@ export function parseMetricsFromUsage(usage: unknown): Record { return metrics; } -export function parseCachedHeader( +function parseCachedHeader( value: string | null | undefined, ): number | undefined { if (!value) { diff --git a/js/src/package-exports.test.ts b/js/src/package-exports.test.ts new file mode 100644 index 000000000..217d05ad7 --- /dev/null +++ b/js/src/package-exports.test.ts @@ -0,0 +1,37 @@ +import { readFileSync } from "node:fs"; +import { describe, expect, test } from "vitest"; + +const packageJson = JSON.parse( + readFileSync(new URL("../package.json", import.meta.url), "utf8"), +) as { exports: Record }; + +describe("package exports", () => { + test("publishes the curated entrypoints", () => { + expect(Object.keys(packageJson.exports).sort()).toEqual( + [ + ".", + "./apply-auto-instrumentation", + "./edge-light", + "./esbuild", + "./hook.mjs", + "./instrumentation", + "./next", + "./node", + "./package.json", + "./rollup", + "./vite", + "./vitest-evals-reporter", + "./webpack", + "./workerd", + ].sort(), + ); + }); + + test("does not publish legacy or implementation entrypoints", () => { + expect(packageJson.exports).not.toHaveProperty("./browser"); + expect(packageJson.exports).not.toHaveProperty("./util"); + expect(packageJson.exports).not.toHaveProperty("./webpack-loader"); + expect(packageJson.exports).not.toHaveProperty("./internal"); + expect(packageJson.exports).not.toHaveProperty("./internal/webpack-loader"); + }); +}); diff --git a/js/src/prompt-schemas.ts b/js/src/prompt-schemas.ts index 46ad5f307..c83c420f0 100644 --- a/js/src/prompt-schemas.ts +++ b/js/src/prompt-schemas.ts @@ -1,9 +1,3 @@ -import { z } from "zod/v3"; -import { - ToolFunctionDefinition as toolFunctionDefinitionSchema, - ChatCompletionMessageParam as chatCompletionMessageParamSchema, - ModelParams as modelParamsSchema, -} from "./generated_types"; import type { ToolFunctionDefinitionType as ToolFunctionDefinition, ChatCompletionMessageParamType, @@ -17,20 +11,6 @@ export type PromptContents = | { prompt: string } | { messages: ChatCompletionMessageParamType[] }; -const internalPromptContentsSchema = z.union([ - z.object({ - prompt: z.string(), - }), - z.object({ - messages: z.array(chatCompletionMessageParamSchema), - }), -]); -export const promptContentsSchema: z.ZodType< - PromptContents, - z.ZodTypeDef, - unknown -> = internalPromptContentsSchema; - export type PromptDefinition = PromptContents & { model: string; params?: ModelParamsType; @@ -38,36 +18,10 @@ export type PromptDefinition = PromptContents & { environments?: string[]; }; -const internalPromptDefinitionSchema = internalPromptContentsSchema.and( - z.object({ - model: z.string(), - params: modelParamsSchema.optional(), - templateFormat: z.enum(["mustache", "nunjucks", "none"]).optional(), - environments: z.array(z.string()).optional(), - }), -); -export const promptDefinitionSchema: z.ZodType< - PromptDefinition, - z.ZodTypeDef, - unknown -> = internalPromptDefinitionSchema; - export type PromptDefinitionWithTools = PromptDefinition & { tools?: ToolFunctionDefinition[]; }; -const internalPromptDefinitionWithToolsSchema = - internalPromptDefinitionSchema.and( - z.object({ - tools: z.array(toolFunctionDefinitionSchema).optional(), - }), - ); -export const promptDefinitionWithToolsSchema: z.ZodType< - PromptDefinitionWithTools, - z.ZodTypeDef, - unknown -> = internalPromptDefinitionWithToolsSchema; - export function promptDefinitionToPromptData( promptDefinition: PromptDefinition, rawTools?: ToolFunctionDefinition[], diff --git a/js/src/propagation.test.ts b/js/src/propagation.test.ts index 17b07dbac..936dfbd84 100644 --- a/js/src/propagation.test.ts +++ b/js/src/propagation.test.ts @@ -10,6 +10,7 @@ import { afterEach, beforeEach, describe, expect, test } from "vitest"; import { _exportsForTestingOnly, + _internalStartSpan, _injectIntoCarrier, extractTraceContextFromHeaders, initLogger, @@ -1026,14 +1027,14 @@ describe("inject / extract / round-trip", () => { root_span_id: pRoot, }).toStr(); - const child = logger.startSpan({ name: "child", parent: slug }); + const child = _internalStartSpan({ name: "child", parent: slug }); expect(child.rootSpanId).toBe(pRoot); expect(child.spanParents).toEqual([pSpan]); child.end(); }); test("legacy parent slug (UUID) linked in hex mode", () => { - const logger = makeLogger(); + makeLogger(); const pSpan = uuidv4(); const pRoot = uuidv4(); const legacySlug = new SpanComponentsV3({ @@ -1044,7 +1045,7 @@ describe("inject / extract / round-trip", () => { root_span_id: pRoot, }).toStr(); - const child = logger.startSpan({ name: "child", parent: legacySlug }); + const child = _internalStartSpan({ name: "child", parent: legacySlug }); // Links to the slug's UUID ids; the child's own span id stays hex. expect(child.rootSpanId).toBe(pRoot); expect(child.spanParents).toEqual([pSpan]); @@ -1064,7 +1065,7 @@ describe("inject / extract / round-trip", () => { root_span_id: pRoot, }).toStr(); - const child = startSpan({ name: "child", parent: legacySlug }); + const child = _internalStartSpan({ name: "child", parent: legacySlug }); expect(child.rootSpanId).toBe(pRoot); expect(child.spanParents).toEqual([pSpan]); expect(child.spanId.length).toBe(16); @@ -1221,7 +1222,7 @@ describe("legacy UUID mode", () => { }); test("legacy parent slug (UUID) linked in legacy mode", () => { - const logger = initLogger({ projectName: "legacy-proj" }); + initLogger({ projectName: "legacy-proj" }); const pSpan = uuidv4(); const pRoot = uuidv4(); const legacySlug = new SpanComponentsV3({ @@ -1232,14 +1233,14 @@ describe("legacy UUID mode", () => { root_span_id: pRoot, }).toStr(); - const child = logger.startSpan({ name: "child", parent: legacySlug }); + const child = _internalStartSpan({ name: "child", parent: legacySlug }); expect(child.rootSpanId).toBe(pRoot); expect(child.spanParents).toEqual([pSpan]); child.end(); }); test("hex parent slug linked in legacy mode", () => { - const logger = initLogger({ projectName: "legacy-proj" }); + initLogger({ projectName: "legacy-proj" }); const pSpan = "00f067aa0ba902b7"; // 8-byte hex const pRoot = "4bf92f3577b34da6a3ce929d0e0e4736"; // 16-byte hex const hexSlug = new SpanComponentsV4({ @@ -1250,7 +1251,7 @@ describe("legacy UUID mode", () => { root_span_id: pRoot, }).toStr(); - const child = logger.startSpan({ name: "child", parent: hexSlug }); + const child = _internalStartSpan({ name: "child", parent: hexSlug }); // Links to the slug's hex ids; the child's own span id stays UUID. expect(child.rootSpanId).toBe(pRoot); expect(child.spanParents).toEqual([pSpan]); diff --git a/js/src/propagation.ts b/js/src/propagation.ts index a8246cce1..1733547aa 100644 --- a/js/src/propagation.ts +++ b/js/src/propagation.ts @@ -26,7 +26,7 @@ const DEFAULT_TRACE_FLAGS = "01"; * not sampled), kept raw so any future flag bits survive a parse -> format * round trip without per-bit handling. */ -export interface ParsedTraceparent { +interface ParsedTraceparent { traceId: string; spanId: string; traceFlags: string; diff --git a/js/src/public-types.test.ts b/js/src/public-types.test.ts index 0583994df..3a37a9cf7 100644 --- a/js/src/public-types.test.ts +++ b/js/src/public-types.test.ts @@ -1,69 +1,84 @@ -import { expectTypeOf, test } from "vitest"; -import { z } from "zod/v3"; +import { expect, expectTypeOf, test } from "vitest"; -import { - AttachmentReference, - braintrustStreamChunkSchema, - logs3OverflowUploadSchema, - promptContentsSchema, - promptDefinitionSchema, - promptDefinitionWithToolsSchema, - type BraintrustStreamChunk, - type EvalParameters, - type Logs3OverflowUpload, - type PromptContents, - type PromptDefinition, - type PromptDefinitionWithTools, +import * as publicExports from "./exports"; +import type { + ContextManager, + ContextParentSpanIds, + CurrentSpanStore, + Dataset, + DurableEvalStore, + EvalScorer, + EvalTask, + LangChainCallbackHandlerOptions, + PropagationContext, + Span, + StartSpanArgs, + TemplateRendererPlugin, + Trace, } from "./exports"; -import type { InferParameters } from "./eval-parameters"; -import type { AttachmentReferenceType } from "./generated_plain_types"; -import type { Prompt } from "./logger"; -import { - spanComponentsV3Schema, - type SpanComponentsV3Data, -} from "../util/span_identifier_v3"; import { - spanComponentsV4Schema, - type SpanComponentsV4Data, -} from "../util/span_identifier_v4"; + BraintrustLangChainCallbackHandler, + init, + initDataset, + withParent, +} from "./exports"; -test("exported validators preserve their public output types", () => { - expectTypeOf< - z.infer - >().toEqualTypeOf(); - expectTypeOf< - z.infer - >().toEqualTypeOf(); - expectTypeOf< - z.infer - >().toEqualTypeOf(); - expectTypeOf< - z.infer - >().toEqualTypeOf(); - expectTypeOf< - z.infer - >().toEqualTypeOf(); +test("exports only intentional extension types", () => { + expectTypeOf().toBeObject(); + expectTypeOf().toBeObject(); + expectTypeOf().toBeObject(); + expectTypeOf().toBeObject(); + expectTypeOf().toBeObject(); + expectTypeOf().toBeObject(); + expectTypeOf(publicExports.configureContextManager).toBeFunction(); + expectTypeOf().toBeObject(); + expectTypeOf().toBeObject(); + expectTypeOf().toBeObject(); + expectTypeOf().toBeObject(); + expectTypeOf(BraintrustLangChainCallbackHandler).toBeConstructibleWith(); expectTypeOf< - z.infer - >().toEqualTypeOf(); - expectTypeOf< - z.infer - >().toEqualTypeOf(); - expectTypeOf< - z.infer - >().toEqualTypeOf(); + EvalTask + >().toBeFunction(); + expectTypeOf>().toBeFunction(); +}); + +test("does not expose runtime schemas or implementation helpers", () => { + for (const name of [ + "AttachmentReference", + "braintrustStreamChunkSchema", + "logs3OverflowUploadSchema", + "promptContentsSchema", + "promptDefinitionSchema", + "promptDefinitionWithToolsSchema", + "SpanImpl", + "IDGenerator", + "_exportsForTestingOnly", + "default", + ]) { + expect(publicExports).not.toHaveProperty(name); + } }); -test("evaluation parameters retain custom schema inference", () => { - const parameters = { - subject: z.string(), - model: { type: "model" as const }, - prompt: { type: "prompt" as const }, - } satisfies EvalParameters; +test("accepts only canonical v4 call shapes", () => { + if (false) { + init({ project: "project" }); + initDataset({ project: "project", dataset: "dataset" }); + + // @ts-expect-error The string-first overload was removed in v4. + init("project"); + // @ts-expect-error The string-first overload was removed in v4. + initDataset("project", { dataset: "dataset" }); + // @ts-expect-error Exported span slugs are internal integration plumbing. + withParent("exported-span", () => undefined); + + const dataset = null as unknown as Dataset; + // @ts-expect-error Dataset records use `expected`, not the legacy `output` alias. + dataset.insert({ input: "input", output: "expected" }); + + const span = null as unknown as Span; + // @ts-expect-error Spans end with `end()`; the `close()` alias was removed. + span.close(); + } - expectTypeOf>().toEqualTypeOf<{ - subject: string; - model: string; - prompt: Prompt; - }>(); + expect(true).toBe(true); }); diff --git a/js/src/sandbox.ts b/js/src/sandbox.ts index cfafa8c98..58c9f45be 100644 --- a/js/src/sandbox.ts +++ b/js/src/sandbox.ts @@ -7,7 +7,7 @@ import { type BraintrustState, _internalGetGlobalState } from "./logger"; * Configuration for a sandbox runtime. * @internal */ -export interface SandboxConfig { +interface SandboxConfig { /** The sandbox provider. Currently only "modal" is supported. */ provider: "modal"; /** Reference to the sandbox snapshot. */ @@ -18,7 +18,7 @@ export interface SandboxConfig { * Options for registering a sandbox function. * @internal */ -export interface RegisterSandboxOptions { +interface RegisterSandboxOptions { /** Group name for the sandbox functions. */ name: string; /** Name of the project to register the sandbox in. */ @@ -47,7 +47,7 @@ export interface RegisterSandboxOptions { * Result of registering a sandbox. * @internal */ -export interface RegisterSandboxResult { +interface RegisterSandboxResult { /** Project ID the sandbox is registered in. */ projectId: string; /** Registered eval functions discovered from this sandbox. */ diff --git a/js/src/template/registry.ts b/js/src/template/registry.ts index 6ebcace61..8c6faa704 100644 --- a/js/src/template/registry.ts +++ b/js/src/template/registry.ts @@ -97,7 +97,7 @@ class TemplatePluginRegistry { } } -export const templateRegistry = new TemplatePluginRegistry(); +const templateRegistry = new TemplatePluginRegistry(); /** * Register a template plugin and optionally activate it diff --git a/js/src/trace.ts b/js/src/trace.ts index d3e6b7610..fd40b6378 100644 --- a/js/src/trace.ts +++ b/js/src/trace.ts @@ -16,7 +16,7 @@ type SpanRecord = any; * Fetcher for spans by root_span_id, using the ObjectFetcher pattern. * Handles pagination automatically via cursor-based iteration. */ -export class SpanFetcher extends ObjectFetcher { +class SpanFetcher extends ObjectFetcher { constructor( objectType: "experiment" | "project_logs" | "playground_logs", private readonly _objectId: string, diff --git a/js/src/workerd/index.ts b/js/src/workerd/index.ts index 7f1f92300..3fa5cdb19 100644 --- a/js/src/workerd/index.ts +++ b/js/src/workerd/index.ts @@ -10,4 +10,3 @@ import { configureWorkerd } from "./config"; configureWorkerd(); export * from "../exports"; -export * as default from "../exports"; diff --git a/js/src/wrappers/ai-sdk/deprecated/BraintrustMiddleware.ts b/js/src/wrappers/ai-sdk/deprecated/BraintrustMiddleware.ts deleted file mode 100644 index d483ed02b..000000000 --- a/js/src/wrappers/ai-sdk/deprecated/BraintrustMiddleware.ts +++ /dev/null @@ -1,493 +0,0 @@ -/* eslint-disable @typescript-eslint/no-explicit-any */ -/* eslint-disable @typescript-eslint/consistent-type-assertions */ -import { SpanTypeAttribute } from "../../../../util/index"; -import { startSpan, type CompiledPrompt } from "../../../logger"; -import { - INSTRUMENTATION_NAMES, - withSpanInstrumentationName, -} from "../../../span-origin"; -import { - extractAnthropicCacheTokens, - finalizeAnthropicTokens, - toNumericMetrics, -} from "../../anthropic-tokens-util"; -import { processInputAttachments } from "../../attachment-utils"; - -function detectProviderFromResult(result: { - providerMetadata?: Record; -}): string | undefined { - if (!result?.providerMetadata) { - return undefined; - } - - const keys = Object.keys(result.providerMetadata); - return keys?.at(0); -} - -function extractModelFromResult(result: { - response?: { modelId?: string }; - request?: { body?: { model?: string } }; -}): string | undefined { - if (result?.response?.modelId) { - return result.response.modelId; - } - - if (result?.request?.body?.model) { - return result.request.body.model; - } - - return undefined; -} - -function extractModelFromWrapGenerateCallback(model: { - modelId?: string; - config?: Record; - specificationVersion?: string; - provider?: string; - supportedUrls?: Record; -}): string | undefined { - return model?.modelId; -} - -function camelToSnake(str: string): string { - return str.replace(/[A-Z]/g, (letter) => `_${letter.toLowerCase()}`); -} - -function extractModelParameters( - params: Record, - excludeKeys: Set, -): Record { - const modelParams: Record = {}; - - for (const [key, value] of Object.entries(params)) { - if (value !== undefined && !excludeKeys.has(key)) { - const snakeKey = camelToSnake(key); - modelParams[snakeKey] = value; - } - } - - return modelParams; -} - -function getNumberProperty(obj: unknown, key: string): number | undefined { - if (!obj || typeof obj !== "object" || !(key in obj)) { - return undefined; - } - const value = Reflect.get(obj, key); - return typeof value === "number" ? value : undefined; -} - -function normalizeUsageMetrics( - usage: unknown, - provider?: string, - providerMetadata?: Record, -): Record { - const metrics: Record = {}; - - // Standard AI SDK usage fields - const inputTokens = getNumberProperty(usage, "inputTokens"); - if (inputTokens !== undefined) { - metrics.prompt_tokens = inputTokens; - } - - const outputTokens = getNumberProperty(usage, "outputTokens"); - if (outputTokens !== undefined) { - metrics.completion_tokens = outputTokens; - } - - const totalTokens = getNumberProperty(usage, "totalTokens"); - if (totalTokens !== undefined) { - metrics.tokens = totalTokens; - } - - const reasoningTokens = getNumberProperty(usage, "reasoningTokens"); - if (reasoningTokens !== undefined) { - metrics.completion_reasoning_tokens = reasoningTokens; - } - - const cachedInputTokens = getNumberProperty(usage, "cachedInputTokens"); - if (cachedInputTokens !== undefined) { - metrics.prompt_cached_tokens = cachedInputTokens; - } - - // Anthropic-specific cache token handling - if (provider === "anthropic") { - const anthropicMetadata = providerMetadata?.anthropic as any; - - if (anthropicMetadata) { - const cacheReadTokens = - getNumberProperty(anthropicMetadata.usage, "cache_read_input_tokens") || - 0; - const cacheCreationTokens = - getNumberProperty( - anthropicMetadata.usage, - "cache_creation_input_tokens", - ) || 0; - - const cacheTokens = extractAnthropicCacheTokens( - cacheReadTokens, - cacheCreationTokens, - ); - Object.assign(metrics, cacheTokens); - - // Use the returned object: finalization can drop cache-creation metrics, - // and merging it back over `metrics` would keep them. - return toNumericMetrics(finalizeAnthropicTokens(metrics)); - } - } - - return metrics; -} - -function normalizeFinishReason(reason: any): string | undefined { - if (typeof reason !== "string") return undefined; - return reason.replace(/-/g, "_"); -} - -function buildAssistantOutputWithToolCalls(result: any, toolCalls: any[]) { - return [ - { - index: 0, - logprobs: null, - finish_reason: - normalizeFinishReason(result?.finishReason) ?? - (toolCalls.length ? "tool_calls" : undefined), - message: { - role: "assistant", - tool_calls: toolCalls.length > 0 ? toolCalls : undefined, - }, - }, - ]; -} - -function extractToolCallsFromSteps(steps: any[] | undefined) { - const toolCalls: any[] = []; - if (!Array.isArray(steps)) return toolCalls; - let idx = 0; - for (const step of steps) { - const blocks: any[] | undefined = (step as any)?.content; - if (!Array.isArray(blocks)) continue; - for (const block of blocks) { - if ( - block && - typeof block === "object" && - (block as any).type === "tool-call" - ) { - toolCalls.push({ - id: (block as any).toolCallId, - type: "function", - index: idx++, - function: { - name: (block as any).toolName, - arguments: - typeof (block as any).input === "string" - ? (block as any).input - : JSON.stringify((block as any).input ?? {}), - }, - }); - } - } - } - return toolCalls; -} - -function extractToolCallsFromBlocks(blocks: any[] | undefined) { - if (!Array.isArray(blocks)) return []; - return extractToolCallsFromSteps([{ content: blocks }] as any); -} - -function extractInput(params: any) { - return params?.prompt ?? params?.messages ?? params?.system; -} - -// Generic middleware interface that works with any AI SDK types -interface LanguageModelV2Middleware { - wrapGenerate?: (options: { - doGenerate: () => any; - doStream: () => any; - params: TCallOptions; - model: TModel; - }) => Promise; - wrapStream?: (options: { - doGenerate: () => any; - doStream: () => any; - params: TCallOptions; - model: TModel; - }) => Promise; -} - -/** - * Configuration options for the AI SDK middleware - */ -interface MiddlewareConfig { - /** Enable debug logging */ - debug?: boolean; - /** Name identifier for the middleware instance */ - name?: string; - /** Span info from loadPrompt for prompt version tracking */ - spanInfo?: CompiledPrompt<"chat">["span_info"]; -} - -// V2-specific exclude keys for extractModelParameters -const V2_EXCLUDE_KEYS = new Set([ - "prompt", // Already captured as input - "system", // Already captured as input - "messages", // Already captured as input - "model", // Already captured in metadata.model - "providerOptions", // Internal AI SDK configuration -]); - -/** - * Creates a Braintrust middleware for AI SDK v2 that automatically traces - * generateText and streamText calls with comprehensive metadata and metrics. - * - * @param config - Configuration options for the middleware - * @returns A middleware object compatible with AI SDK v2's wrapLanguageModel - * - * @deprecated The new `wrapAISDK` replaces this middleware. - * - * @example - * ```typescript - * import { wrapLanguageModel } from "ai"; - * import { openai } from "@ai-sdk/openai"; - * import { BraintrustMiddleware } from "braintrust"; - * - * const model = wrapLanguageModel({ - * model: openai("gpt-4"), - * middleware: BraintrustMiddleware({ debug: true, name: "MyMiddleware" }) - * }); - * ``` - * - */ -export function BraintrustMiddleware( - config: MiddlewareConfig = {}, -): LanguageModelV2Middleware { - return { - wrapGenerate: async ({ - doGenerate, - params, - model: modelFromWrapGenerate, - }) => { - // Extract and process input attachments - const rawInput = extractInput(params); - const processedInput = processInputAttachments(rawInput); - - const spanArgs = { - name: config.spanInfo?.name || "ai-sdk.doGenerate", - spanAttributes: { - type: SpanTypeAttribute.LLM, - ...(config.spanInfo?.spanAttributes || {}), - }, - event: { - input: processedInput, - metadata: { - ...extractModelParameters(params, V2_EXCLUDE_KEYS), - ...(config.spanInfo?.metadata || {}), - }, - }, - }; - - const span = startSpan( - withSpanInstrumentationName(spanArgs, INSTRUMENTATION_NAMES.AI_SDK), - ); - - try { - const result = await doGenerate(); - - const metadata: Record = {}; - - const provider = detectProviderFromResult(result); - if (provider !== undefined) { - metadata.provider = provider; - } - - if (result.finishReason !== undefined) { - metadata.finish_reason = result.finishReason; - } - - const model = extractModelFromResult(result); - if (model !== undefined) { - metadata.model = model; - } else if (modelFromWrapGenerate) { - // Use the model from the wrapGenerate call if it's not in the result - const modelId = extractModelFromWrapGenerateCallback( - modelFromWrapGenerate, - ); - if (modelId) { - metadata.model = modelId; - } - } - - let toolCalls = extractToolCallsFromSteps((result as any)?.steps); - if (!toolCalls || toolCalls.length === 0) { - toolCalls = extractToolCallsFromBlocks((result as any)?.content); - } - - span.log({ - output: - toolCalls.length > 0 - ? buildAssistantOutputWithToolCalls(result, toolCalls) - : (result as any)?.content, - metadata, - metrics: normalizeUsageMetrics( - result.usage, - provider, - result.providerMetadata, - ), - }); - - return result; - } catch (error) { - span.log({ - error: error instanceof Error ? error.message : String(error), - }); - throw error; - } finally { - span.end(); - } - }, - wrapStream: async ({ doStream, params }) => { - // Extract and process input attachments - const rawInput = extractInput(params); - const processedInput = processInputAttachments(rawInput); - - const spanArgs = { - name: config.spanInfo?.name || "ai-sdk.doStream", - spanAttributes: { - type: SpanTypeAttribute.LLM, - ...(config.spanInfo?.spanAttributes || {}), - }, - event: { - input: processedInput, - metadata: { - ...extractModelParameters(params, V2_EXCLUDE_KEYS), - ...(config.spanInfo?.metadata || {}), - }, - }, - }; - - const span = startSpan( - withSpanInstrumentationName(spanArgs, INSTRUMENTATION_NAMES.AI_SDK), - ); - - try { - const { stream, ...rest } = await doStream(); - - const textChunks: string[] = []; - const toolBlocks: any[] = []; - let finalUsage: unknown = {}; - let finalFinishReason: unknown = undefined; - let providerMetadata: Record = {}; - - const transformStream = new TransformStream({ - transform(chunk: any, controller: any) { - try { - // Collect text deltas - if (chunk.type === "text-delta" && chunk.delta) { - textChunks.push(chunk.delta); - } - - // Collect tool call/result blocks for formatting later - if (chunk.type === "tool-call" || chunk.type === "tool-result") { - toolBlocks.push(chunk); - } - - // Capture final metadata - if (chunk.type === "finish") { - finalFinishReason = chunk.finishReason; - finalUsage = chunk.usage || {}; - providerMetadata = chunk.providerMetadata || {}; - } - - controller.enqueue(chunk); - } catch (error) { - // Log stream processing error - span.log({ - error: error instanceof Error ? error.message : String(error), - }); - span.end(); - controller.error(error); - } - }, - - flush() { - try { - // Log the final aggregated result when stream completes - const generatedText = textChunks.join(""); - let output: unknown = generatedText - ? [{ type: "text", text: generatedText }] - : []; - - // Create a result object for provider detection - const resultForDetection = { - providerMetadata, - response: rest.response, - ...rest, - finishReason: finalFinishReason, - }; - - const metadata: Record = {}; - - const provider = detectProviderFromResult(resultForDetection); - if (provider !== undefined) { - metadata.provider = provider; - } - - if (finalFinishReason !== undefined) { - metadata.finish_reason = finalFinishReason; - } - - const model = extractModelFromResult(resultForDetection); - if (model !== undefined) { - metadata.model = model; - } - - // If tool calls streamed, prefer assistant tool_calls output - if (toolBlocks.length > 0) { - const toolCalls = extractToolCallsFromSteps([ - { content: toolBlocks }, - ] as any); - if (toolCalls.length > 0) { - output = buildAssistantOutputWithToolCalls( - resultForDetection, - toolCalls, - ); - } - } - - span.log({ - output, - metadata, - metrics: normalizeUsageMetrics( - finalUsage, - provider, - providerMetadata, - ), - }); - - span.end(); - } catch (error) { - // Log flush error - span.log({ - error: error instanceof Error ? error.message : String(error), - }); - span.end(); - throw error; - } - }, - }); - - return { - stream: stream.pipeThrough(transformStream), - ...rest, - }; - } catch (error) { - span.log({ - error: error instanceof Error ? error.message : String(error), - }); - span.end(); - throw error; - } - }, - }; -} diff --git a/js/src/wrappers/ai-sdk/deprecated/wrapAISDKModel.ts b/js/src/wrappers/ai-sdk/deprecated/wrapAISDKModel.ts deleted file mode 100644 index 705048346..000000000 --- a/js/src/wrappers/ai-sdk/deprecated/wrapAISDKModel.ts +++ /dev/null @@ -1,362 +0,0 @@ -/* eslint-disable @typescript-eslint/consistent-type-assertions */ -/* eslint-disable @typescript-eslint/no-explicit-any */ -import { startSpan } from "../../../logger"; -import { - INSTRUMENTATION_NAMES, - withSpanInstrumentationName, -} from "../../../span-origin"; -import { getCurrentUnixTimestamp, isEmpty } from "../../../util"; -import { - LEGACY_CACHED_HEADER, - parseCachedHeader, - X_CACHED_HEADER, -} from "../../oai"; - -/** - * Wrap an ai-sdk model (created with `.chat()`, `.completion()`, etc.) to add tracing. If Braintrust is - * not configured, this is a no-op - * - * @param model - * @deprecated You should use the new `wrapAISDK`. - * @returns The wrapped object. - */ -export function wrapAISDKModel(model: T): T { - const m = model as any; - if ( - m?.specificationVersion === "v1" && - typeof m?.provider === "string" && - typeof m?.modelId === "string" - ) { - return new BraintrustLanguageModelWrapper(m) as any as T; - } else { - // eslint-disable-next-line no-restricted-properties -- preserving intentional console usage. - console.warn("Unsupported AI SDK model. Not wrapping."); - return model; - } -} - -class BraintrustLanguageModelWrapper { - supportsUrl?: (url: URL) => boolean; - - constructor(private model: any) { - // Only define supportsUrl if the model has this method - if (typeof this.model.supportsUrl === "function") { - this.supportsUrl = (url: URL) => this.model.supportsUrl!(url); - } - } - - get specificationVersion() { - return this.model.specificationVersion; - } - - get provider(): string { - return this.model.provider; - } - - get modelId(): string { - return this.model.modelId; - } - - get defaultObjectGenerationMode() { - return this.model.defaultObjectGenerationMode; - } - - get supportsImageUrls(): boolean | undefined { - return this.model.supportsImageUrls; - } - - get supportsStructuredOutputs(): boolean | undefined { - return this.model.supportsStructuredOutputs; - } - - // For the first cut, do not support custom span_info arguments. We can - // propagate those via async local storage - async doGenerate(options: any) { - const span = startSpan( - withSpanInstrumentationName( - { - name: "Chat Completion", - spanAttributes: { - type: "llm", - }, - }, - INSTRUMENTATION_NAMES.AI_SDK, - ), - ); - const { prompt, mode, ...rest } = options; - const startTime = getCurrentUnixTimestamp(); - - try { - const ret = await this.model.doGenerate(options); - span.log({ - input: postProcessPrompt(prompt), - metadata: { - model: this.modelId, - ...rest, - ...("tools" in mode && mode.tools - ? { tools: convertTools(mode.tools) } - : "tool" in mode && mode.tool - ? { tools: convertTools([mode.tool]) } - : {}), - }, - output: postProcessOutput(ret.text, ret.toolCalls, ret.finishReason), - metrics: { - time_to_first_token: getCurrentUnixTimestamp() - startTime, - tokens: !isEmpty(ret.usage) - ? ret.usage.promptTokens + ret.usage.completionTokens - : undefined, - prompt_tokens: ret.usage?.promptTokens, - completion_tokens: ret.usage?.completionTokens, - cached: parseCachedHeader( - ret.rawResponse?.headers?.[X_CACHED_HEADER] ?? - ret.rawResponse?.headers?.[LEGACY_CACHED_HEADER], - ), - }, - }); - return ret; - } finally { - span.end(); - } - } - - async doStream(options: any) { - const { prompt, mode, ...rest } = options; - const startTime = getCurrentUnixTimestamp(); - - const span = startSpan( - withSpanInstrumentationName( - { - name: "Chat Completion", - spanAttributes: { - type: "llm", - }, - }, - INSTRUMENTATION_NAMES.AI_SDK, - ), - ); - - span.log({ - input: postProcessPrompt(prompt), - metadata: { - model: this.modelId, - ...rest, - ...("tools" in mode && mode.tools - ? { tools: convertTools(mode.tools) } - : "tool" in mode && mode.tool - ? { tools: convertTools([mode.tool]) } - : {}), - }, - }); - - let ended = false; - const end = () => { - if (!ended) { - span.end(); - ended = true; - } - }; - - try { - const ret = await this.model.doStream(options); - - let time_to_first_token: number | undefined = undefined; - let usage: - | { - promptTokens: number; - completionTokens: number; - } - | undefined = undefined; - let fullText: string | undefined = undefined; - const toolCalls: Record = {}; - let finishReason: any | undefined = undefined; - return { - ...ret, - stream: ret.stream.pipeThrough( - new TransformStream({ - transform(chunk, controller) { - if (time_to_first_token === undefined) { - time_to_first_token = getCurrentUnixTimestamp() - startTime; - span.log({ metrics: { time_to_first_token } }); - } - switch (chunk.type) { - case "text-delta": - if (fullText === undefined) { - fullText = ""; - } - fullText += chunk.textDelta; - break; - case "tool-call": - toolCalls[chunk.toolCallId] = { - toolCallType: chunk.toolCallType, - toolCallId: chunk.toolCallId, - toolName: chunk.toolName, - args: chunk.args, - }; - break; - case "tool-call-delta": - if (toolCalls[chunk.toolCallId] === undefined) { - toolCalls[chunk.toolCallId] = { - toolCallType: chunk.toolCallType, - toolCallId: chunk.toolCallId, - toolName: chunk.toolName, - args: "", - }; - } - toolCalls[chunk.toolCallId].args += chunk.argsTextDelta; - break; - case "finish": - usage = chunk.usage; - finishReason = chunk.finishReason; - break; - } - - controller.enqueue(chunk); - }, - async flush(controller) { - span.log({ - output: postProcessOutput( - fullText, - Object.keys(toolCalls).length > 0 - ? Object.values(toolCalls) - : undefined, - finishReason!, - ), - metrics: { - time_to_first_token, - tokens: !isEmpty(usage) - ? usage.promptTokens + usage.completionTokens - : undefined, - prompt_tokens: usage?.promptTokens, - completion_tokens: usage?.completionTokens, - cached: parseCachedHeader( - ret.rawResponse?.headers?.[X_CACHED_HEADER] ?? - ret.rawResponse?.headers?.[LEGACY_CACHED_HEADER], - ), - }, - }); - end(); - controller.terminate(); - }, - }), - ), - }; - } finally { - end(); - } - } -} - -function convertTools(tools: Array): any[] { - return tools.map((tool) => { - const { type: _, ...rest } = tool; - return { - type: tool.type, - function: rest, - }; - }) as any[]; -} - -function postProcessPrompt(prompt: any): any[] { - return prompt.flatMap((message: any) => { - switch (message.role) { - case "system": - return [ - { - role: "system", - content: message.content, - }, - ]; - case "assistant": - const textPart = message.content.find( - (part: any) => part.type === "text", - ) as any | undefined; - const toolCallParts = message.content.filter( - (part: any) => part.type === "tool-call", - ) as any[]; - return [ - { - role: "assistant", - content: textPart?.text, - ...(toolCallParts.length > 0 - ? { - tool_calls: toolCallParts.map((part) => ({ - id: part.toolCallId, - function: { - name: part.toolName, - arguments: JSON.stringify(part.args), - }, - type: "function" as const, - })), - } - : {}), - }, - ]; - case "user": - return [ - { - role: "user", - content: message.content.map((part: any): any => { - switch (part.type) { - case "text": - return { - type: "text", - text: part.text, - ...(part.providerMetadata - ? { providerMetadata: part.providerMetadata } - : {}), - }; - case "image": - return { - type: "image_url", - image_url: { - url: part.image.toString(), - ...(part.providerMetadata - ? { providerMetadata: part.providerMetadata } - : {}), - }, - }; - default: - // We don't support files directly but also don't want to block them from being logged - return part as any; - } - }), - }, - ]; - case "tool": - return message.content.map((part: any) => ({ - role: "tool", - tool_call_id: part.toolCallId, - content: JSON.stringify(part.result), - })); - } - }); -} - -function postProcessOutput( - text: string | undefined, - toolCalls: any[] | undefined, - finishReason: any, -) { - return [ - { - index: 0, - message: { - role: "assistant", - content: text ?? "", - ...(toolCalls && toolCalls.length > 0 - ? { - tool_calls: toolCalls.map((toolCall) => ({ - id: toolCall.toolCallId, - function: { - name: toolCall.toolName, - arguments: toolCall.args, - }, - type: "function" as const, - })), - } - : {}), - }, - finish_reason: finishReason, - }, - ]; -} diff --git a/js/src/wrappers/ai-sdk/harness-agent-context.ts b/js/src/wrappers/ai-sdk/harness-agent-context.ts index f17f2a442..0aa4fbb5d 100644 --- a/js/src/wrappers/ai-sdk/harness-agent-context.ts +++ b/js/src/wrappers/ai-sdk/harness-agent-context.ts @@ -2,8 +2,8 @@ import iso from "../../isomorph"; import type { IsoAsyncLocalStorage, IsoTracingChannel } from "../../isomorph"; import { _internalGetGlobalState, + _internalStartSpan, currentSpan, - startSpan, updateSpan, type Span, type StartSpanArgs, @@ -359,9 +359,10 @@ export function startHarnessTurnChildSpan( args, INSTRUMENTATION_NAMES.AI_SDK, ); + const { parent: _ignoredParent, ...publicSpanArgs } = spanArgs; return typeof parent === "string" - ? startSpan({ ...spanArgs, parent }) - : parent.startSpan(spanArgs); + ? _internalStartSpan({ ...spanArgs, parent }) + : parent.startSpan(publicSpanArgs); } export function updateHarnessTurn( diff --git a/js/src/wrappers/ai-sdk/index.ts b/js/src/wrappers/ai-sdk/index.ts index b2058cd16..f9e1d86f3 100644 --- a/js/src/wrappers/ai-sdk/index.ts +++ b/js/src/wrappers/ai-sdk/index.ts @@ -1,6 +1,2 @@ export { wrapAISDK, wrapAgentClass } from "./ai-sdk"; export { braintrustAISDKTelemetry } from "./telemetry"; - -// TODO: remove in the next major release -export { wrapAISDKModel } from "./deprecated/wrapAISDKModel"; -export { BraintrustMiddleware } from "./deprecated/BraintrustMiddleware"; diff --git a/js/src/wrappers/exports.test.ts b/js/src/wrappers/exports.test.ts deleted file mode 100644 index 66850a3b3..000000000 --- a/js/src/wrappers/exports.test.ts +++ /dev/null @@ -1,38 +0,0 @@ -import { describe, it, expect } from "vitest"; -import { BraintrustMiddleware } from "./ai-sdk"; - -describe("ai-sdk exports", () => { - it("should always export BraintrustMiddleware as a function", () => { - expect(typeof BraintrustMiddleware).toBe("function"); - }); - - it("BraintrustMiddleware should return an object with wrapGenerate and wrapStream", () => { - const result = BraintrustMiddleware({}); - expect(result).toHaveProperty("wrapGenerate"); - expect(result).toHaveProperty("wrapStream"); - expect(typeof result.wrapGenerate).toBe("function"); - expect(typeof result.wrapStream).toBe("function"); - }); - - it("should handle conditional imports gracefully", () => { - // Test that imports don't throw errors regardless of AI SDK version - expect(() => { - const middleware = BraintrustMiddleware({ debug: true }); - - // Should be able to call the functions without errors - const { wrapGenerate, wrapStream } = middleware; - - expect(wrapGenerate).toBeDefined(); - expect(wrapStream).toBeDefined(); - }).not.toThrow(); - }); - - it("should export middleware functions that can be instantiated", () => { - const middleware = BraintrustMiddleware({}); - const { wrapGenerate, wrapStream } = middleware; - - // Should be functions that can be called (we don't test actual execution due to logger dependencies) - expect(typeof wrapGenerate).toBe("function"); - expect(typeof wrapStream).toBe("function"); - }); -}); diff --git a/js/src/wrappers/langchain/callback-handler.ts b/js/src/wrappers/langchain/callback-handler.ts index 51b073173..27bf90f28 100644 --- a/js/src/wrappers/langchain/callback-handler.ts +++ b/js/src/wrappers/langchain/callback-handler.ts @@ -89,13 +89,14 @@ export class BraintrustLangChainCallbackHandler { INSTRUMENTATION_NAMES.LANGCHAIN, ); - let span = parentSpan.startSpan(spanArgs); + const { parent: _ignoredParent, ...publicSpanArgs } = spanArgs; + let span = parentSpan.startSpan(publicSpanArgs); if ( !Object.is(this.options.logger, NOOP_SPAN) && Object.is(span, NOOP_SPAN) ) { - span = initLogger().startSpan(spanArgs); + span = initLogger().startSpan(publicSpanArgs); } this.spans.set(runId, span); diff --git a/js/src/wrappers/mastra.ts b/js/src/wrappers/mastra.ts index b3ff3b419..4ecaaacbe 100644 --- a/js/src/wrappers/mastra.ts +++ b/js/src/wrappers/mastra.ts @@ -75,7 +75,7 @@ interface MastraTracingEvent { } /** Subset of the `ObservabilityExporter` contract from `@mastra/core`. */ -export interface MastraObservabilityExporter { +interface MastraObservabilityExporter { name: string; init?(options: unknown): void; __setLogger?(logger: unknown): void; @@ -328,11 +328,12 @@ export class BraintrustObservabilityExporter implements MastraObservabilityExpor } } + const { parent: _ignoredParent, ...publicArgs } = args; const span = parentRecord - ? parentRecord.span.startSpan(args) + ? parentRecord.span.startSpan(publicArgs) : this.capturedParent - ? this.capturedParent.startSpan(args) - : startSpan(args); + ? this.capturedParent.startSpan(publicArgs) + : startSpan(publicArgs); const record: SpanRecord = { span, hasLoggedInput: false }; this.logPayload(record, exported); @@ -408,40 +409,3 @@ export class BraintrustObservabilityExporter implements MastraObservabilityExpor function logExporterError(err: unknown): void { debugLogger.warn("Mastra exporter failure:", err); } - -/** - * @deprecated Mastra is now instrumented through its own `ObservabilityExporter` - * contract instead of by wrapping the agent. This function does nothing and - * will be removed in the next major release. - * - * To capture Mastra spans in Braintrust, do one of: - * - * - **Auto-instrumentation**: run your app with - * `node --import braintrust/hook.mjs`. The loader installs - * `BraintrustObservabilityExporter` into every `new Mastra(...)` - * automatically. - * - **Manual wiring**: pass the exporter yourself: - * - * ```ts - * import { Mastra } from "@mastra/core"; - * import { Observability } from "@mastra/observability"; - * import { BraintrustObservabilityExporter } from "braintrust"; - * - * const mastra = new Mastra({ - * observability: new Observability({ - * configs: { - * default: { - * serviceName: "my-service", - * exporters: [new BraintrustObservabilityExporter()], - * }, - * }, - * }), - * }); - * ``` - */ -export function wrapMastraAgent( - agent: T, - _options?: { name?: string; span_name?: string }, -): T { - return agent; -} diff --git a/js/src/wrappers/node-test/suite.ts b/js/src/wrappers/node-test/suite.ts index c2ee794df..80cf58dae 100644 --- a/js/src/wrappers/node-test/suite.ts +++ b/js/src/wrappers/node-test/suite.ts @@ -1,4 +1,4 @@ -import { initExperiment, type Experiment } from "../../logger"; +import { init, type Experiment } from "../../logger"; import { runTracedEval } from "../shared/traced-eval"; import { summarizeAndFlush } from "../shared/flush"; import type { @@ -40,7 +40,8 @@ export function initNodeTestSuite(config: NodeTestSuiteConfig): NodeTestSuite { const experimentName = config.experimentName || `${config.projectName}-${new Date().toISOString()}`; - experiment = initExperiment(config.projectName, { + experiment = init({ + project: config.projectName, experiment: experimentName, }); return experiment; diff --git a/js/src/wrappers/node-test/test-helpers.ts b/js/src/wrappers/node-test/test-helpers.ts index 0047bf5ac..a455eae66 100644 --- a/js/src/wrappers/node-test/test-helpers.ts +++ b/js/src/wrappers/node-test/test-helpers.ts @@ -19,14 +19,12 @@ export async function setupNodeTestEnv(): Promise { await _exportsForTestingOnly.simulateLoginForTests(); const bgLogger = _exportsForTestingOnly.useTestBackgroundLogger(); - vi.spyOn(logger, "initExperiment").mockImplementation( - (projectName: string, options?: any) => { - return _exportsForTestingOnly.initTestExperiment( - options?.experiment || "test-experiment", - projectName, - ); - }, - ); + vi.spyOn(logger, "init").mockImplementation((options) => { + return _exportsForTestingOnly.initTestExperiment( + options.experiment || "test-experiment", + options.project ?? options.projectId ?? "test-project", + ); + }); return bgLogger; } diff --git a/js/src/wrappers/oai.ts b/js/src/wrappers/oai.ts index 743460b7e..0599787e0 100644 --- a/js/src/wrappers/oai.ts +++ b/js/src/wrappers/oai.ts @@ -1,10 +1,5 @@ /* eslint-disable @typescript-eslint/no-explicit-any */ import type { CompiledPrompt } from "../logger"; -import { - LEGACY_CACHED_HEADER, - parseCachedHeader, - X_CACHED_HEADER, -} from "../openai-utils"; import { responsesProxy } from "./oai_responses"; import type { ArgsOf, @@ -72,7 +67,7 @@ globalThis.__inherited_braintrust_wrap_openai = wrapOpenAI; type OpenAILike = OpenAIV4Client; -export function wrapOpenAIv4(openai: T): T { +function wrapOpenAIv4(openai: T): T { // eslint-disable-next-line @typescript-eslint/consistent-type-assertions const typedOpenai = openai as OpenAIV4Client; // Recover `this` for fallback methods so private fields and internal slots @@ -238,8 +233,6 @@ function wrapBetaChatCompletionStream

( }; } -export { LEGACY_CACHED_HEADER, parseCachedHeader, X_CACHED_HEADER }; - function wrapChatCompletion< P extends OpenAIChatCreateParams, C extends OpenAIChatCompletion | OpenAIChatStream, diff --git a/js/src/wrappers/vitest-evals/reporter.test.ts b/js/src/wrappers/vitest-evals/reporter.test.ts index 7b7385306..ccff70440 100644 --- a/js/src/wrappers/vitest-evals/reporter.test.ts +++ b/js/src/wrappers/vitest-evals/reporter.test.ts @@ -16,7 +16,7 @@ beforeAll(async () => { await _exportsForTestingOnly.simulateLoginForTests(); backgroundLogger = _exportsForTestingOnly.useTestBackgroundLogger(); - vi.spyOn(logger, "initExperiment").mockImplementation( + vi.spyOn(logger, "init").mockImplementation( (projectOrOptions: string | any, options?: any) => { const experimentOptions = typeof projectOrOptions === "string" ? options : projectOrOptions; diff --git a/js/src/wrappers/vitest-evals/reporter.ts b/js/src/wrappers/vitest-evals/reporter.ts index 71ba251e3..ad3de38be 100644 --- a/js/src/wrappers/vitest-evals/reporter.ts +++ b/js/src/wrappers/vitest-evals/reporter.ts @@ -1,11 +1,6 @@ import type { Reporter, TestCase, TestModule, Vitest } from "vitest/node"; import { SpanTypeAttribute, isObject } from "../../../util"; -import { - initExperiment, - logError, - type Experiment, - type Span, -} from "../../logger"; +import { init, logError, type Experiment, type Span } from "../../logger"; import { configureNode } from "../../node/config"; import { summarizeAndFlush } from "../shared/flush"; @@ -202,7 +197,7 @@ export default class BraintrustVitestEvalsReporter implements Reporter { const experimentName = this.options.experimentName ?? `vitest-evals-${new Date().toISOString()}`; - this.experiment = initExperiment({ + this.experiment = init({ ...(projectId ? { projectId } : { project: projectName }), experiment: experimentName, metadata: this.options.metadata, diff --git a/js/src/wrappers/vitest/context-manager.ts b/js/src/wrappers/vitest/context-manager.ts index 03cb41799..88b6198b3 100644 --- a/js/src/wrappers/vitest/context-manager.ts +++ b/js/src/wrappers/vitest/context-manager.ts @@ -12,7 +12,7 @@ import type { Dataset, Experiment } from "../../logger"; * - flushPromise/flushResolved: Coordination for experiment flushing */ export interface VitestExperimentContext { - dataset: Dataset | undefined; + dataset: Dataset | undefined; experiment: Experiment; datasetExamples: Map; parent?: VitestExperimentContext; // Link to parent describe @@ -65,7 +65,7 @@ class VitestContextManager { } createChildContext( - dataset: Dataset | undefined, + dataset: Dataset | undefined, experiment: Experiment, ): VitestExperimentContext { const parent = this.getCurrentContext(); diff --git a/js/src/wrappers/vitest/vitest-wrapper-example.test.ts b/js/src/wrappers/vitest/vitest-wrapper-example.test.ts index c646c0fdf..452e6e8ca 100644 --- a/js/src/wrappers/vitest/vitest-wrapper-example.test.ts +++ b/js/src/wrappers/vitest/vitest-wrapper-example.test.ts @@ -11,7 +11,7 @@ await _exportsForTestingOnly.simulateLoginForTests(); const moduleBackgroundLogger = _exportsForTestingOnly.useTestBackgroundLogger(); vitest.vi - .spyOn(logger, "initExperiment") + .spyOn(logger, "init") .mockImplementation((projectName: string, options?: any) => { return _exportsForTestingOnly.initTestExperiment( options?.experiment || "test-experiment", diff --git a/js/src/wrappers/vitest/vitest-wrapper-span.test.ts b/js/src/wrappers/vitest/vitest-wrapper-span.test.ts index 47cbcbdbe..a7995b20a 100644 --- a/js/src/wrappers/vitest/vitest-wrapper-span.test.ts +++ b/js/src/wrappers/vitest/vitest-wrapper-span.test.ts @@ -11,7 +11,7 @@ await _exportsForTestingOnly.simulateLoginForTests(); const moduleBackgroundLogger = _exportsForTestingOnly.useTestBackgroundLogger(); vitest.vi - .spyOn(logger, "initExperiment") + .spyOn(logger, "init") .mockImplementation((projectName: string, options?: any) => { return _exportsForTestingOnly.initTestExperiment( options?.experiment || "test-experiment", diff --git a/js/src/wrappers/vitest/vitest-wrapper.test.ts b/js/src/wrappers/vitest/vitest-wrapper.test.ts index 6154b21b0..d3d7bed6f 100644 --- a/js/src/wrappers/vitest/vitest-wrapper.test.ts +++ b/js/src/wrappers/vitest/vitest-wrapper.test.ts @@ -32,13 +32,13 @@ _exportsForTestingOnly.setInitialTestState(); await _exportsForTestingOnly.simulateLoginForTests(); _exportsForTestingOnly.useTestBackgroundLogger(); -// ✅ STEP 2: Mock initDataset and initExperiment to avoid network calls +// ✅ STEP 2: Mock initDataset and init to avoid network calls vi.spyOn(logger, "initDataset").mockReturnValue({ insert: vi.fn(() => "test-example-id"), } as any); -const initExperimentSpy = vi - .spyOn(logger, "initExperiment") +const initSpy = vi + .spyOn(logger, "init") .mockImplementation((projectOrOptions: string | any, options?: any) => { const experimentOptions = typeof projectOrOptions === "string" ? options : projectOrOptions; @@ -451,11 +451,11 @@ describe("Project selection", () => { } beforeEach(() => { - initExperimentSpy.mockClear(); + initSpy.mockClear(); _resetContextManager(); }); - test("wrapVitest passes projectId to initExperiment", async () => { + test("wrapVitest passes projectId to init", async () => { const bt = wrapVitest(makeFakeVitestMethods(), { projectId: "project-id-123", displaySummary: false, @@ -465,7 +465,7 @@ describe("Project selection", () => { bt.test("uses project id", async () => undefined); }); - expect(initExperimentSpy).toHaveBeenCalledWith( + expect(initSpy).toHaveBeenCalledWith( expect.objectContaining({ projectId: "project-id-123", experiment: expect.stringMatching(/^project id suite-/), @@ -484,15 +484,12 @@ describe("Project selection", () => { bt.test("uses project id", async () => undefined); }); - expect(initExperimentSpy).toHaveBeenCalledWith( + expect(initSpy).toHaveBeenCalledWith( expect.objectContaining({ projectId: "project-id-123", }), ); - expect(initExperimentSpy).not.toHaveBeenCalledWith( - "project-name", - expect.anything(), - ); + expect(initSpy).not.toHaveBeenCalledWith("project-name", expect.anything()); }); test("wrapVitest preserves projectName behavior when projectId is absent", async () => { @@ -505,9 +502,9 @@ describe("Project selection", () => { bt.test("uses project name", async () => undefined); }); - expect(initExperimentSpy).toHaveBeenCalledWith( - "project-name", + expect(initSpy).toHaveBeenCalledWith( expect.objectContaining({ + project: "project-name", experiment: expect.stringMatching(/^project name suite-/), }), ); @@ -522,9 +519,9 @@ describe("Project selection", () => { bt.test("uses suite name", async () => undefined); }); - expect(initExperimentSpy).toHaveBeenCalledWith( - "fallback suite", + expect(initSpy).toHaveBeenCalledWith( expect.objectContaining({ + project: "fallback suite", experiment: expect.stringMatching(/^fallback suite-/), }), ); diff --git a/js/src/wrappers/vitest/wrapper.ts b/js/src/wrappers/vitest/wrapper.ts index 4be2adf5b..c0f0ee381 100644 --- a/js/src/wrappers/vitest/wrapper.ts +++ b/js/src/wrappers/vitest/wrapper.ts @@ -1,4 +1,4 @@ -import { initExperiment } from "../../logger"; +import { init } from "../../logger"; import type { TestConfig, TestContext, @@ -215,11 +215,12 @@ export function wrapDescribe( const experimentName = `${suiteName}-${new Date().toISOString()}`; const experiment = config.projectId - ? initExperiment({ + ? init({ projectId: config.projectId, experiment: experimentName, }) - : initExperiment(config.projectName || suiteName, { + : init({ + project: config.projectName || suiteName, experiment: experimentName, }); diff --git a/js/tests/api-compatibility/api-compatibility.test.ts b/js/tests/api-compatibility/api-compatibility.test.ts index 3941302be..6241d23d6 100644 --- a/js/tests/api-compatibility/api-compatibility.test.ts +++ b/js/tests/api-compatibility/api-compatibility.test.ts @@ -20,7 +20,7 @@ import type { Options } from "tsup"; * ## How It Works * * 1. Downloads the latest published version from npm - * 2. Extracts the .d.ts files for each entrypoint (main, browser, util) + * 2. Extracts the .d.ts files for each built entrypoint * 3. Parses both published and current .d.ts files using TypeScript Compiler API * 4. Compares exported symbols (functions, classes, interfaces, types, etc.) * 5. Fails if breaking changes are detected in non-major version bumps @@ -77,10 +77,10 @@ async function getEntrypointsFromTsupConfig(): Promise< entrypoints.push({ name, typesPath }); } } else if (typeof entry === "object") { - // entry is a record like { main: 'src/index.ts' } - for (const [key, entryFile] of Object.entries(entry)) { + // Object entry keys determine the emitted path, including nested paths. + for (const key of Object.keys(entry)) { const name = key; - const typesPath = getTypesPath(String(entryFile), outDir); + const typesPath = path.join(outDir, `${key}.d.ts`); entrypoints.push({ name, typesPath }); } } @@ -97,10 +97,8 @@ function getEntrypointName(entryFile: string, outDir: string): string { // Map common patterns to friendly names if (entryFile.includes("src/node/index.ts")) return "main"; - if (entryFile.includes("src/browser/index.ts")) return "browser"; if (entryFile.includes("src/edge-light/index.ts")) return "edge-light"; if (entryFile.includes("src/workerd/index.ts")) return "workerd"; - if (entryFile.includes("util/index.ts")) return "util"; // Default to basename return basename; @@ -2951,13 +2949,14 @@ describe("API Compatibility", () => { test("keeps public declarations free of expanded Zod schema graphs", () => { const declarationRoot = path.join(__dirname, "..", ".."); const publicDeclarationPaths = [ - "dist/index.d.ts", - "dist/browser.d.ts", - "util/dist/index.d.ts", + ...new Set(ENTRYPOINTS.map(({ typesPath }) => typesPath)), ]; for (const declarationTypesPath of publicDeclarationPaths) { const declarationPath = path.join(declarationRoot, declarationTypesPath); + if (!fs.existsSync(declarationPath)) { + continue; + } const declaration = fs.readFileSync(declarationPath, "utf8"); expect(declaration).not.toMatch(/z\.infer { "promptDefinitionSchema", "promptDefinitionWithToolsSchema", ]) { - expect(mainDeclaration).toMatch( - new RegExp(`declare const ${schemaName}: z\\.ZodType<`), - ); - } - - const utilDeclaration = fs.readFileSync( - path.join(declarationRoot, "util/dist/index.d.ts"), - "utf8", - ); - for (const schemaName of [ - "spanComponentsV3Schema", - "spanComponentsV4Schema", - "spanObjectTypeV3EnumSchema", - ]) { - expect(utilDeclaration).toMatch( - new RegExp(`declare const ${schemaName}: z\\.ZodType<`), + expect(mainDeclaration).not.toMatch( + new RegExp(`(?:declare|export) const ${schemaName}(?::|\\s*=)`), ); } }); diff --git a/js/tests/auto-instrumentations/error-handling.test.ts b/js/tests/auto-instrumentations/error-handling.test.ts index 2c31588a4..7bbccfe75 100644 --- a/js/tests/auto-instrumentations/error-handling.test.ts +++ b/js/tests/auto-instrumentations/error-handling.test.ts @@ -73,7 +73,7 @@ describe("Error Handling", () => { fs.writeFileSync(entryPoint, testCode); - const { esbuildPlugin } = + const { braintrustEsbuildPlugin } = await import("../../src/auto-instrumentations/bundler/esbuild.js"); await esbuild.build({ @@ -82,7 +82,7 @@ describe("Error Handling", () => { write: true, outfile, format: "esm", - plugins: [esbuildPlugin({ browser: false })], + plugins: [braintrustEsbuildPlugin({ browser: false })], logLevel: "error", absWorkingDir: fixturesDir, preserveSymlinks: true, @@ -133,7 +133,7 @@ describe("Error Handling", () => { fs.writeFileSync(entryPoint, testCode); - const { esbuildPlugin } = + const { braintrustEsbuildPlugin } = await import("../../src/auto-instrumentations/bundler/esbuild.js"); await esbuild.build({ @@ -142,7 +142,7 @@ describe("Error Handling", () => { write: true, outfile, format: "esm", - plugins: [esbuildPlugin({ browser: false })], + plugins: [braintrustEsbuildPlugin({ browser: false })], logLevel: "error", absWorkingDir: fixturesDir, preserveSymlinks: true, @@ -184,7 +184,7 @@ describe("Error Handling", () => { fs.writeFileSync(entryPoint, testCode); - const { esbuildPlugin } = + const { braintrustEsbuildPlugin } = await import("../../src/auto-instrumentations/bundler/esbuild.js"); await esbuild.build({ @@ -193,7 +193,7 @@ describe("Error Handling", () => { write: true, outfile, format: "esm", - plugins: [esbuildPlugin({ browser: false })], + plugins: [braintrustEsbuildPlugin({ browser: false })], logLevel: "error", absWorkingDir: fixturesDir, preserveSymlinks: true, @@ -228,7 +228,7 @@ describe("Error Handling", () => { fs.writeFileSync(entryPoint, testCode); - const { esbuildPlugin } = + const { braintrustEsbuildPlugin } = await import("../../src/auto-instrumentations/bundler/esbuild.js"); await esbuild.build({ @@ -237,7 +237,7 @@ describe("Error Handling", () => { write: true, outfile, format: "esm", - plugins: [esbuildPlugin({ browser: false })], + plugins: [braintrustEsbuildPlugin({ browser: false })], logLevel: "error", absWorkingDir: fixturesDir, preserveSymlinks: true, @@ -281,7 +281,7 @@ describe("Error Handling", () => { fs.writeFileSync(entryPoint, testCode); - const { esbuildPlugin } = + const { braintrustEsbuildPlugin } = await import("../../src/auto-instrumentations/bundler/esbuild.js"); await esbuild.build({ @@ -290,7 +290,7 @@ describe("Error Handling", () => { write: true, outfile, format: "esm", - plugins: [esbuildPlugin({ browser: false })], + plugins: [braintrustEsbuildPlugin({ browser: false })], logLevel: "error", absWorkingDir: fixturesDir, preserveSymlinks: true, @@ -336,7 +336,7 @@ describe("Error Handling", () => { fs.writeFileSync(entryPoint, testCode); - const { esbuildPlugin } = + const { braintrustEsbuildPlugin } = await import("../../src/auto-instrumentations/bundler/esbuild.js"); await esbuild.build({ @@ -345,7 +345,7 @@ describe("Error Handling", () => { write: true, outfile, format: "esm", - plugins: [esbuildPlugin({ browser: false })], + plugins: [braintrustEsbuildPlugin({ browser: false })], logLevel: "error", absWorkingDir: fixturesDir, preserveSymlinks: true, @@ -400,7 +400,7 @@ describe("Error Handling", () => { fs.writeFileSync(entryPoint, testCode); - const { esbuildPlugin } = + const { braintrustEsbuildPlugin } = await import("../../src/auto-instrumentations/bundler/esbuild.js"); await esbuild.build({ @@ -409,7 +409,7 @@ describe("Error Handling", () => { write: true, outfile, format: "esm", - plugins: [esbuildPlugin({ browser: false })], + plugins: [braintrustEsbuildPlugin({ browser: false })], logLevel: "error", absWorkingDir: fixturesDir, preserveSymlinks: true, diff --git a/js/tests/auto-instrumentations/event-content.test.ts b/js/tests/auto-instrumentations/event-content.test.ts index 5a2fdd100..81acdee76 100644 --- a/js/tests/auto-instrumentations/event-content.test.ts +++ b/js/tests/auto-instrumentations/event-content.test.ts @@ -75,7 +75,7 @@ describe("Event Content Validation", () => { fs.writeFileSync(entryPoint, testCode); - const { esbuildPlugin } = + const { braintrustEsbuildPlugin } = await import("../../src/auto-instrumentations/bundler/esbuild.js"); await esbuild.build({ @@ -84,7 +84,7 @@ describe("Event Content Validation", () => { write: true, outfile, format: "esm", - plugins: [esbuildPlugin({ browser: false })], + plugins: [braintrustEsbuildPlugin({ browser: false })], logLevel: "error", absWorkingDir: fixturesDir, preserveSymlinks: true, @@ -149,7 +149,7 @@ describe("Event Content Validation", () => { fs.writeFileSync(entryPoint, testCode); - const { esbuildPlugin } = + const { braintrustEsbuildPlugin } = await import("../../src/auto-instrumentations/bundler/esbuild.js"); await esbuild.build({ @@ -158,7 +158,7 @@ describe("Event Content Validation", () => { write: true, outfile, format: "esm", - plugins: [esbuildPlugin({ browser: false })], + plugins: [braintrustEsbuildPlugin({ browser: false })], logLevel: "error", absWorkingDir: fixturesDir, preserveSymlinks: true, @@ -216,7 +216,7 @@ describe("Event Content Validation", () => { fs.writeFileSync(entryPoint, testCode); - const { esbuildPlugin } = + const { braintrustEsbuildPlugin } = await import("../../src/auto-instrumentations/bundler/esbuild.js"); await esbuild.build({ @@ -225,7 +225,7 @@ describe("Event Content Validation", () => { write: true, outfile, format: "esm", - plugins: [esbuildPlugin({ browser: false })], + plugins: [braintrustEsbuildPlugin({ browser: false })], logLevel: "error", absWorkingDir: fixturesDir, preserveSymlinks: true, @@ -296,7 +296,7 @@ describe("Event Content Validation", () => { fs.writeFileSync(entryPoint, testCode); - const { esbuildPlugin } = + const { braintrustEsbuildPlugin } = await import("../../src/auto-instrumentations/bundler/esbuild.js"); await esbuild.build({ @@ -305,7 +305,7 @@ describe("Event Content Validation", () => { write: true, outfile, format: "esm", - plugins: [esbuildPlugin({ browser: false })], + plugins: [braintrustEsbuildPlugin({ browser: false })], logLevel: "error", absWorkingDir: fixturesDir, preserveSymlinks: true, @@ -357,7 +357,7 @@ describe("Event Content Validation", () => { fs.writeFileSync(entryPoint, testCode); - const { esbuildPlugin } = + const { braintrustEsbuildPlugin } = await import("../../src/auto-instrumentations/bundler/esbuild.js"); await esbuild.build({ @@ -366,7 +366,7 @@ describe("Event Content Validation", () => { write: true, outfile, format: "esm", - plugins: [esbuildPlugin({ browser: false })], + plugins: [braintrustEsbuildPlugin({ browser: false })], logLevel: "error", absWorkingDir: fixturesDir, preserveSymlinks: true, @@ -411,7 +411,7 @@ describe("Event Content Validation", () => { fs.writeFileSync(entryPoint, testCode); - const { esbuildPlugin } = + const { braintrustEsbuildPlugin } = await import("../../src/auto-instrumentations/bundler/esbuild.js"); await esbuild.build({ @@ -420,7 +420,7 @@ describe("Event Content Validation", () => { write: true, outfile, format: "esm", - plugins: [esbuildPlugin({ browser: false })], + plugins: [braintrustEsbuildPlugin({ browser: false })], logLevel: "error", absWorkingDir: fixturesDir, preserveSymlinks: true, @@ -467,7 +467,7 @@ describe("Event Content Validation", () => { fs.writeFileSync(entryPoint, testCode); - const { esbuildPlugin } = + const { braintrustEsbuildPlugin } = await import("../../src/auto-instrumentations/bundler/esbuild.js"); await esbuild.build({ @@ -476,7 +476,7 @@ describe("Event Content Validation", () => { write: true, outfile, format: "esm", - plugins: [esbuildPlugin({ browser: false })], + plugins: [braintrustEsbuildPlugin({ browser: false })], logLevel: "error", absWorkingDir: fixturesDir, preserveSymlinks: true, @@ -525,7 +525,7 @@ describe("Event Content Validation", () => { fs.writeFileSync(entryPoint, testCode); - const { esbuildPlugin } = + const { braintrustEsbuildPlugin } = await import("../../src/auto-instrumentations/bundler/esbuild.js"); await esbuild.build({ @@ -534,7 +534,7 @@ describe("Event Content Validation", () => { write: true, outfile, format: "esm", - plugins: [esbuildPlugin({ browser: false })], + plugins: [braintrustEsbuildPlugin({ browser: false })], logLevel: "error", absWorkingDir: fixturesDir, preserveSymlinks: true, @@ -652,7 +652,7 @@ describe("Event Content Validation", () => { fs.writeFileSync(entryPoint, testCode); - const { esbuildPlugin } = + const { braintrustEsbuildPlugin } = await import("../../src/auto-instrumentations/bundler/esbuild.js"); await esbuild.build({ @@ -661,7 +661,7 @@ describe("Event Content Validation", () => { write: true, outfile, format: "esm", - plugins: [esbuildPlugin({ browser: false })], + plugins: [braintrustEsbuildPlugin({ browser: false })], logLevel: "error", absWorkingDir: fixturesDir, preserveSymlinks: true, @@ -727,7 +727,7 @@ describe("Event Content Validation", () => { fs.writeFileSync(entryPoint, testCode); - const { esbuildPlugin } = + const { braintrustEsbuildPlugin } = await import("../../src/auto-instrumentations/bundler/esbuild.js"); await esbuild.build({ @@ -736,7 +736,7 @@ describe("Event Content Validation", () => { write: true, outfile, format: "esm", - plugins: [esbuildPlugin({ browser: false })], + plugins: [braintrustEsbuildPlugin({ browser: false })], logLevel: "error", absWorkingDir: fixturesDir, preserveSymlinks: true, @@ -831,7 +831,7 @@ describe("Event Content Validation", () => { fs.writeFileSync(entryPoint, testCode); - const { esbuildPlugin } = + const { braintrustEsbuildPlugin } = await import("../../src/auto-instrumentations/bundler/esbuild.js"); await esbuild.build({ @@ -840,7 +840,7 @@ describe("Event Content Validation", () => { write: true, outfile, format: "esm", - plugins: [esbuildPlugin({ browser: false })], + plugins: [braintrustEsbuildPlugin({ browser: false })], logLevel: "error", absWorkingDir: fixturesDir, preserveSymlinks: true, @@ -938,7 +938,7 @@ describe("Event Content Validation", () => { fs.writeFileSync(entryPoint, testCode); - const { esbuildPlugin } = + const { braintrustEsbuildPlugin } = await import("../../src/auto-instrumentations/bundler/esbuild.js"); await esbuild.build({ @@ -947,7 +947,7 @@ describe("Event Content Validation", () => { write: true, outfile, format: "esm", - plugins: [esbuildPlugin({ browser: false })], + plugins: [braintrustEsbuildPlugin({ browser: false })], logLevel: "error", absWorkingDir: fixturesDir, preserveSymlinks: true, @@ -1045,7 +1045,7 @@ describe("Event Content Validation", () => { fs.writeFileSync(entryPoint, testCode); - const { esbuildPlugin } = + const { braintrustEsbuildPlugin } = await import("../../src/auto-instrumentations/bundler/esbuild.js"); await esbuild.build({ @@ -1054,7 +1054,7 @@ describe("Event Content Validation", () => { write: true, outfile, format: "esm", - plugins: [esbuildPlugin({ browser: false })], + plugins: [braintrustEsbuildPlugin({ browser: false })], logLevel: "error", absWorkingDir: fixturesDir, preserveSymlinks: true, @@ -1132,7 +1132,7 @@ describe("Event Content Validation", () => { fs.writeFileSync(entryPoint, testCode); - const { esbuildPlugin } = + const { braintrustEsbuildPlugin } = await import("../../src/auto-instrumentations/bundler/esbuild.js"); await esbuild.build({ @@ -1141,7 +1141,7 @@ describe("Event Content Validation", () => { write: true, outfile, format: "esm", - plugins: [esbuildPlugin({ browser: false })], + plugins: [braintrustEsbuildPlugin({ browser: false })], logLevel: "error", absWorkingDir: fixturesDir, preserveSymlinks: true, @@ -1233,7 +1233,7 @@ describe("Event Content Validation", () => { fs.writeFileSync(entryPoint, testCode); - const { esbuildPlugin } = + const { braintrustEsbuildPlugin } = await import("../../src/auto-instrumentations/bundler/esbuild.js"); await esbuild.build({ @@ -1242,7 +1242,7 @@ describe("Event Content Validation", () => { write: true, outfile, format: "esm", - plugins: [esbuildPlugin({ browser: false })], + plugins: [braintrustEsbuildPlugin({ browser: false })], logLevel: "error", absWorkingDir: fixturesDir, preserveSymlinks: true, @@ -1340,7 +1340,7 @@ describe("Event Content Validation", () => { fs.writeFileSync(entryPoint, testCode); - const { esbuildPlugin } = + const { braintrustEsbuildPlugin } = await import("../../src/auto-instrumentations/bundler/esbuild.js"); await esbuild.build({ @@ -1349,7 +1349,7 @@ describe("Event Content Validation", () => { write: true, outfile, format: "esm", - plugins: [esbuildPlugin({ browser: false })], + plugins: [braintrustEsbuildPlugin({ browser: false })], logLevel: "error", absWorkingDir: fixturesDir, preserveSymlinks: true, @@ -1416,7 +1416,7 @@ describe("Event Content Validation", () => { fs.writeFileSync(entryPoint, testCode); - const { esbuildPlugin } = + const { braintrustEsbuildPlugin } = await import("../../src/auto-instrumentations/bundler/esbuild.js"); await esbuild.build({ @@ -1425,7 +1425,7 @@ describe("Event Content Validation", () => { write: true, outfile, format: "esm", - plugins: [esbuildPlugin({ browser: false })], + plugins: [braintrustEsbuildPlugin({ browser: false })], logLevel: "error", absWorkingDir: fixturesDir, preserveSymlinks: true, @@ -1514,7 +1514,7 @@ describe("Event Content Validation", () => { fs.writeFileSync(entryPoint, testCode); - const { esbuildPlugin } = + const { braintrustEsbuildPlugin } = await import("../../src/auto-instrumentations/bundler/esbuild.js"); await esbuild.build({ @@ -1523,7 +1523,7 @@ describe("Event Content Validation", () => { write: true, outfile, format: "esm", - plugins: [esbuildPlugin({ browser: false })], + plugins: [braintrustEsbuildPlugin({ browser: false })], logLevel: "error", absWorkingDir: fixturesDir, preserveSymlinks: true, @@ -1603,7 +1603,7 @@ describe("Event Content Validation", () => { fs.writeFileSync(entryPoint, testCode); - const { esbuildPlugin } = + const { braintrustEsbuildPlugin } = await import("../../src/auto-instrumentations/bundler/esbuild.js"); await esbuild.build({ @@ -1612,7 +1612,7 @@ describe("Event Content Validation", () => { write: true, outfile, format: "esm", - plugins: [esbuildPlugin({ browser: false })], + plugins: [braintrustEsbuildPlugin({ browser: false })], logLevel: "error", absWorkingDir: fixturesDir, preserveSymlinks: true, @@ -1699,7 +1699,7 @@ describe("Event Content Validation", () => { fs.writeFileSync(entryPoint, testCode); - const { esbuildPlugin } = + const { braintrustEsbuildPlugin } = await import("../../src/auto-instrumentations/bundler/esbuild.js"); await esbuild.build({ @@ -1708,7 +1708,7 @@ describe("Event Content Validation", () => { write: true, outfile, format: "esm", - plugins: [esbuildPlugin({ browser: false })], + plugins: [braintrustEsbuildPlugin({ browser: false })], logLevel: "error", absWorkingDir: fixturesDir, preserveSymlinks: true, diff --git a/js/tests/auto-instrumentations/function-behavior.test.ts b/js/tests/auto-instrumentations/function-behavior.test.ts index 1797252b5..0881a6d53 100644 --- a/js/tests/auto-instrumentations/function-behavior.test.ts +++ b/js/tests/auto-instrumentations/function-behavior.test.ts @@ -63,7 +63,7 @@ describe("Function Behavior Preservation", () => { fs.writeFileSync(entryPoint, testCode); - const { esbuildPlugin } = + const { braintrustEsbuildPlugin } = await import("../../src/auto-instrumentations/bundler/esbuild.js"); await esbuild.build({ @@ -72,7 +72,7 @@ describe("Function Behavior Preservation", () => { write: true, outfile, format: "esm", - plugins: [esbuildPlugin({ browser: false })], + plugins: [braintrustEsbuildPlugin({ browser: false })], logLevel: "error", absWorkingDir: fixturesDir, preserveSymlinks: true, @@ -131,7 +131,7 @@ describe("Function Behavior Preservation", () => { fs.writeFileSync(entryPoint, testCode); - const { esbuildPlugin } = + const { braintrustEsbuildPlugin } = await import("../../src/auto-instrumentations/bundler/esbuild.js"); await esbuild.build({ @@ -140,7 +140,7 @@ describe("Function Behavior Preservation", () => { write: true, outfile, format: "esm", - plugins: [esbuildPlugin({ browser: false })], + plugins: [braintrustEsbuildPlugin({ browser: false })], logLevel: "error", absWorkingDir: fixturesDir, preserveSymlinks: true, @@ -178,7 +178,7 @@ describe("Function Behavior Preservation", () => { fs.writeFileSync(entryPoint, testCode); - const { esbuildPlugin } = + const { braintrustEsbuildPlugin } = await import("../../src/auto-instrumentations/bundler/esbuild.js"); await esbuild.build({ @@ -187,7 +187,7 @@ describe("Function Behavior Preservation", () => { write: true, outfile, format: "esm", - plugins: [esbuildPlugin({ browser: false })], + plugins: [braintrustEsbuildPlugin({ browser: false })], logLevel: "error", absWorkingDir: fixturesDir, preserveSymlinks: true, @@ -230,7 +230,7 @@ describe("Function Behavior Preservation", () => { fs.writeFileSync(entryPoint, testCode); - const { esbuildPlugin } = + const { braintrustEsbuildPlugin } = await import("../../src/auto-instrumentations/bundler/esbuild.js"); await esbuild.build({ @@ -239,7 +239,7 @@ describe("Function Behavior Preservation", () => { write: true, outfile, format: "esm", - plugins: [esbuildPlugin({ browser: false })], + plugins: [braintrustEsbuildPlugin({ browser: false })], logLevel: "error", absWorkingDir: fixturesDir, preserveSymlinks: true, @@ -284,7 +284,7 @@ describe("Function Behavior Preservation", () => { fs.writeFileSync(entryPoint, testCode); - const { esbuildPlugin } = + const { braintrustEsbuildPlugin } = await import("../../src/auto-instrumentations/bundler/esbuild.js"); await esbuild.build({ @@ -293,7 +293,7 @@ describe("Function Behavior Preservation", () => { write: true, outfile, format: "esm", - plugins: [esbuildPlugin({ browser: false })], + plugins: [braintrustEsbuildPlugin({ browser: false })], logLevel: "error", absWorkingDir: fixturesDir, preserveSymlinks: true, @@ -338,7 +338,7 @@ describe("Function Behavior Preservation", () => { fs.writeFileSync(entryPoint, testCode); - const { esbuildPlugin } = + const { braintrustEsbuildPlugin } = await import("../../src/auto-instrumentations/bundler/esbuild.js"); await esbuild.build({ @@ -347,7 +347,7 @@ describe("Function Behavior Preservation", () => { write: true, outfile, format: "esm", - plugins: [esbuildPlugin({ browser: false })], + plugins: [braintrustEsbuildPlugin({ browser: false })], logLevel: "error", absWorkingDir: fixturesDir, preserveSymlinks: true, @@ -392,7 +392,7 @@ describe("Function Behavior Preservation", () => { fs.writeFileSync(entryPoint, testCode); - const { esbuildPlugin } = + const { braintrustEsbuildPlugin } = await import("../../src/auto-instrumentations/bundler/esbuild.js"); await esbuild.build({ @@ -401,7 +401,7 @@ describe("Function Behavior Preservation", () => { write: true, outfile, format: "esm", - plugins: [esbuildPlugin({ browser: false })], + plugins: [braintrustEsbuildPlugin({ browser: false })], logLevel: "error", absWorkingDir: fixturesDir, preserveSymlinks: true, @@ -438,7 +438,7 @@ describe("Function Behavior Preservation", () => { fs.writeFileSync(entryPoint, testCode); - const { esbuildPlugin } = + const { braintrustEsbuildPlugin } = await import("../../src/auto-instrumentations/bundler/esbuild.js"); await esbuild.build({ @@ -447,7 +447,7 @@ describe("Function Behavior Preservation", () => { write: true, outfile, format: "esm", - plugins: [esbuildPlugin({ browser: false })], + plugins: [braintrustEsbuildPlugin({ browser: false })], logLevel: "error", absWorkingDir: fixturesDir, preserveSymlinks: true, @@ -489,7 +489,7 @@ describe("Function Behavior Preservation", () => { fs.writeFileSync(entryPoint, testCode); - const { esbuildPlugin } = + const { braintrustEsbuildPlugin } = await import("../../src/auto-instrumentations/bundler/esbuild.js"); await esbuild.build({ @@ -498,7 +498,7 @@ describe("Function Behavior Preservation", () => { write: true, outfile, format: "esm", - plugins: [esbuildPlugin({ browser: false })], + plugins: [braintrustEsbuildPlugin({ browser: false })], logLevel: "error", absWorkingDir: fixturesDir, preserveSymlinks: true, diff --git a/js/tests/auto-instrumentations/multiple-instrumentations.test.ts b/js/tests/auto-instrumentations/multiple-instrumentations.test.ts index 2b48ff0a6..bb53cbe60 100644 --- a/js/tests/auto-instrumentations/multiple-instrumentations.test.ts +++ b/js/tests/auto-instrumentations/multiple-instrumentations.test.ts @@ -101,7 +101,7 @@ describe("Multiple Instrumentations", () => { fs.writeFileSync(entryPoint, testCode); - const { esbuildPlugin } = + const { braintrustEsbuildPlugin } = await import("../../src/auto-instrumentations/bundler/esbuild.js"); await esbuild.build({ @@ -110,7 +110,7 @@ describe("Multiple Instrumentations", () => { write: true, outfile, format: "esm", - plugins: [esbuildPlugin({ browser: false })], + plugins: [braintrustEsbuildPlugin({ browser: false })], logLevel: "error", absWorkingDir: fixturesDir, preserveSymlinks: true, @@ -169,7 +169,7 @@ describe("Multiple Instrumentations", () => { fs.writeFileSync(entryPoint, testCode); - const { esbuildPlugin } = + const { braintrustEsbuildPlugin } = await import("../../src/auto-instrumentations/bundler/esbuild.js"); await esbuild.build({ @@ -178,7 +178,7 @@ describe("Multiple Instrumentations", () => { write: true, outfile, format: "esm", - plugins: [esbuildPlugin({ browser: false })], + plugins: [braintrustEsbuildPlugin({ browser: false })], logLevel: "error", absWorkingDir: fixturesDir, preserveSymlinks: true, @@ -234,7 +234,7 @@ describe("Multiple Instrumentations", () => { fs.writeFileSync(entryPoint, testCode); - const { esbuildPlugin } = + const { braintrustEsbuildPlugin } = await import("../../src/auto-instrumentations/bundler/esbuild.js"); await esbuild.build({ @@ -243,7 +243,7 @@ describe("Multiple Instrumentations", () => { write: true, outfile, format: "esm", - plugins: [esbuildPlugin({ browser: false })], + plugins: [braintrustEsbuildPlugin({ browser: false })], logLevel: "error", absWorkingDir: fixturesDir, preserveSymlinks: true, @@ -314,7 +314,7 @@ describe("Multiple Instrumentations", () => { fs.writeFileSync(entryPoint, testCode); - const { esbuildPlugin } = + const { braintrustEsbuildPlugin } = await import("../../src/auto-instrumentations/bundler/esbuild.js"); await esbuild.build({ @@ -323,7 +323,7 @@ describe("Multiple Instrumentations", () => { write: true, outfile, format: "esm", - plugins: [esbuildPlugin({ browser: false })], + plugins: [braintrustEsbuildPlugin({ browser: false })], logLevel: "error", absWorkingDir: fixturesDir, preserveSymlinks: true, @@ -402,7 +402,7 @@ describe("Multiple Instrumentations", () => { fs.writeFileSync(entryPoint, testCode); - const { esbuildPlugin } = + const { braintrustEsbuildPlugin } = await import("../../src/auto-instrumentations/bundler/esbuild.js"); // Add custom instrumentation config @@ -427,7 +427,7 @@ describe("Multiple Instrumentations", () => { outfile, format: "esm", plugins: [ - esbuildPlugin({ + braintrustEsbuildPlugin({ browser: false, instrumentations: [customConfig], }), @@ -488,7 +488,7 @@ describe("Multiple Instrumentations", () => { fs.writeFileSync(entryPoint, testCode); - const { esbuildPlugin } = + const { braintrustEsbuildPlugin } = await import("../../src/auto-instrumentations/bundler/esbuild.js"); await esbuild.build({ @@ -497,7 +497,7 @@ describe("Multiple Instrumentations", () => { write: true, outfile, format: "esm", - plugins: [esbuildPlugin({ browser: false })], + plugins: [braintrustEsbuildPlugin({ browser: false })], logLevel: "error", absWorkingDir: fixturesDir, preserveSymlinks: true, diff --git a/js/tests/auto-instrumentations/next-config.test.ts b/js/tests/auto-instrumentations/next-config.test.ts index 91d96d401..7ebefe3ce 100644 --- a/js/tests/auto-instrumentations/next-config.test.ts +++ b/js/tests/auto-instrumentations/next-config.test.ts @@ -1,14 +1,14 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; vi.mock("../../src/auto-instrumentations/bundler/webpack.js", () => ({ - webpackPlugin: vi.fn((options: unknown) => ({ + braintrustWebpackPlugin: vi.fn((options: unknown) => ({ apply: () => {}, name: "braintrust-test-webpack-plugin", options, })), })); -import { webpackPlugin } from "../../src/auto-instrumentations/bundler/webpack.js"; +import { braintrustWebpackPlugin } from "../../src/auto-instrumentations/bundler/webpack.js"; import { wrapNextjsConfigWithBraintrust } from "../../src/auto-instrumentations/bundler/next.js"; const originalArgv = [...process.argv]; @@ -45,7 +45,7 @@ describe("wrapNextjsConfigWithBraintrust", () => { const result = config.webpack({ plugins: [] }, { isServer: false }); expect(userWebpack).toHaveBeenCalledOnce(); - expect(webpackPlugin).toHaveBeenCalledWith({ browser: true }); + expect(braintrustWebpackPlugin).toHaveBeenCalledWith({ browser: true }); expect(result.plugins).toHaveLength(2); expect(result.plugins[0]).toEqual({ name: "user-plugin" }); expect(result.plugins[1].options).toEqual({ browser: true }); @@ -183,8 +183,8 @@ describe("wrapNextjsConfigWithBraintrust", () => { }, { resolve: (specifier: string) => { - if (specifier === "braintrust/webpack-loader") { - return "/braintrust/webpack-loader.cjs"; + if (specifier === "braintrust/package.json") { + return "/braintrust/package.json"; } throw new Error(`Cannot resolve module ${specifier}`); diff --git a/js/tests/auto-instrumentations/runtime-execution.test.ts b/js/tests/auto-instrumentations/runtime-execution.test.ts index eb882dd3f..991826798 100644 --- a/js/tests/auto-instrumentations/runtime-execution.test.ts +++ b/js/tests/auto-instrumentations/runtime-execution.test.ts @@ -63,7 +63,7 @@ describe("Runtime Execution of Bundled Code", () => { fs.writeFileSync(entryPoint, testCode); - const { esbuildPlugin } = + const { braintrustEsbuildPlugin } = await import("../../src/auto-instrumentations/bundler/esbuild.js"); await esbuild.build({ @@ -72,7 +72,7 @@ describe("Runtime Execution of Bundled Code", () => { write: true, outfile, format: "esm", - plugins: [esbuildPlugin({ browser: false })], + plugins: [braintrustEsbuildPlugin({ browser: false })], logLevel: "error", absWorkingDir: fixturesDir, preserveSymlinks: true, @@ -122,7 +122,7 @@ describe("Runtime Execution of Bundled Code", () => { fs.writeFileSync(entryPoint, testCode); - const { esbuildPlugin } = + const { braintrustEsbuildPlugin } = await import("../../src/auto-instrumentations/bundler/esbuild.js"); await esbuild.build({ @@ -131,7 +131,7 @@ describe("Runtime Execution of Bundled Code", () => { write: true, outfile, format: "esm", - plugins: [esbuildPlugin({ browser: false })], + plugins: [braintrustEsbuildPlugin({ browser: false })], logLevel: "error", absWorkingDir: fixturesDir, preserveSymlinks: true, @@ -188,7 +188,7 @@ describe("Runtime Execution of Bundled Code", () => { fs.writeFileSync(entryPoint, testCode); - const { esbuildPlugin } = + const { braintrustEsbuildPlugin } = await import("../../src/auto-instrumentations/bundler/esbuild.js"); await esbuild.build({ @@ -197,7 +197,7 @@ describe("Runtime Execution of Bundled Code", () => { write: true, outfile, format: "esm", - plugins: [esbuildPlugin({ browser: false })], + plugins: [braintrustEsbuildPlugin({ browser: false })], logLevel: "error", absWorkingDir: fixturesDir, preserveSymlinks: true, @@ -233,7 +233,7 @@ describe("Runtime Execution of Bundled Code", () => { fs.writeFileSync(entryPoint, testCode); - const { vitePlugin } = + const { braintrustVitePlugin } = await import("../../src/auto-instrumentations/bundler/vite.js"); await viteBuild({ @@ -248,7 +248,7 @@ describe("Runtime Execution of Bundled Code", () => { emptyOutDir: true, minify: false, }, - plugins: [vitePlugin({ browser: false })], + plugins: [braintrustVitePlugin({ browser: false })], logLevel: "error", resolve: { preserveSymlinks: true, @@ -292,7 +292,7 @@ describe("Runtime Execution of Bundled Code", () => { fs.writeFileSync(entryPoint, testCode); - const { vitePlugin } = + const { braintrustVitePlugin } = await import("../../src/auto-instrumentations/bundler/vite.js"); await viteBuild({ @@ -307,7 +307,7 @@ describe("Runtime Execution of Bundled Code", () => { emptyOutDir: true, minify: false, }, - plugins: [vitePlugin({ browser: false })], + plugins: [braintrustVitePlugin({ browser: false })], logLevel: "error", resolve: { preserveSymlinks: true, @@ -344,7 +344,7 @@ describe("Runtime Execution of Bundled Code", () => { fs.writeFileSync(entryPoint, testCode); - const { rollupPlugin } = + const { braintrustRollupPlugin } = await import("../../src/auto-instrumentations/bundler/rollup.js"); // Simple resolver plugin @@ -363,7 +363,7 @@ describe("Runtime Execution of Bundled Code", () => { const bundle = await rollup({ input: entryPoint, - plugins: [resolverPlugin, rollupPlugin({ browser: false })], + plugins: [resolverPlugin, braintrustRollupPlugin({ browser: false })], external: [], preserveSymlinks: true, }); @@ -413,7 +413,7 @@ describe("Runtime Execution of Bundled Code", () => { fs.writeFileSync(entryPoint, testCode); - const { rollupPlugin } = + const { braintrustRollupPlugin } = await import("../../src/auto-instrumentations/bundler/rollup.js"); const resolverPlugin = { @@ -431,7 +431,7 @@ describe("Runtime Execution of Bundled Code", () => { const bundle = await rollup({ input: entryPoint, - plugins: [resolverPlugin, rollupPlugin({ browser: false })], + plugins: [resolverPlugin, braintrustRollupPlugin({ browser: false })], external: [], preserveSymlinks: true, }); @@ -476,7 +476,7 @@ describe("Runtime Execution of Bundled Code", () => { fs.writeFileSync(entryPoint, testCode); - const { esbuildPlugin } = + const { braintrustEsbuildPlugin } = await import("../../src/auto-instrumentations/bundler/esbuild.js"); await esbuild.build({ @@ -485,7 +485,7 @@ describe("Runtime Execution of Bundled Code", () => { write: true, outfile, format: "esm", - plugins: [esbuildPlugin({ browser: false })], + plugins: [braintrustEsbuildPlugin({ browser: false })], logLevel: "error", absWorkingDir: fixturesDir, preserveSymlinks: true, @@ -530,7 +530,7 @@ describe("Runtime Execution of Bundled Code", () => { fs.writeFileSync(entryPoint, testCode); - const { esbuildPlugin } = + const { braintrustEsbuildPlugin } = await import("../../src/auto-instrumentations/bundler/esbuild.js"); await esbuild.build({ @@ -539,7 +539,7 @@ describe("Runtime Execution of Bundled Code", () => { write: true, outfile, format: "esm", - plugins: [esbuildPlugin({ browser: false })], + plugins: [braintrustEsbuildPlugin({ browser: false })], logLevel: "error", absWorkingDir: fixturesDir, preserveSymlinks: true, diff --git a/js/tests/auto-instrumentations/streaming-and-responses.test.ts b/js/tests/auto-instrumentations/streaming-and-responses.test.ts index e994de00e..aafd66bf5 100644 --- a/js/tests/auto-instrumentations/streaming-and-responses.test.ts +++ b/js/tests/auto-instrumentations/streaming-and-responses.test.ts @@ -155,7 +155,7 @@ describe("Streaming Methods and Responses API", () => { fs.writeFileSync(entryPoint, testCode); - const { esbuildPlugin } = + const { braintrustEsbuildPlugin } = await import("../../src/auto-instrumentations/bundler/esbuild.js"); await esbuild.build({ @@ -164,7 +164,7 @@ describe("Streaming Methods and Responses API", () => { write: true, outfile, format: "esm", - plugins: [esbuildPlugin({ browser: false })], + plugins: [braintrustEsbuildPlugin({ browser: false })], logLevel: "error", absWorkingDir: fixturesDir, preserveSymlinks: true, @@ -216,7 +216,7 @@ describe("Streaming Methods and Responses API", () => { fs.writeFileSync(entryPoint, testCode); - const { esbuildPlugin } = + const { braintrustEsbuildPlugin } = await import("../../src/auto-instrumentations/bundler/esbuild.js"); await esbuild.build({ @@ -225,7 +225,7 @@ describe("Streaming Methods and Responses API", () => { write: true, outfile, format: "esm", - plugins: [esbuildPlugin({ browser: false })], + plugins: [braintrustEsbuildPlugin({ browser: false })], logLevel: "error", absWorkingDir: fixturesDir, preserveSymlinks: true, @@ -310,7 +310,7 @@ describe("Streaming Methods and Responses API", () => { fs.writeFileSync(entryPoint, testCode); - const { esbuildPlugin } = + const { braintrustEsbuildPlugin } = await import("../../src/auto-instrumentations/bundler/esbuild.js"); await esbuild.build({ @@ -319,7 +319,7 @@ describe("Streaming Methods and Responses API", () => { write: true, outfile, format: "esm", - plugins: [esbuildPlugin({ browser: false })], + plugins: [braintrustEsbuildPlugin({ browser: false })], logLevel: "error", absWorkingDir: fixturesDir, preserveSymlinks: true, @@ -400,7 +400,7 @@ describe("Streaming Methods and Responses API", () => { fs.writeFileSync(entryPoint, testCode); - const { esbuildPlugin } = + const { braintrustEsbuildPlugin } = await import("../../src/auto-instrumentations/bundler/esbuild.js"); await esbuild.build({ @@ -409,7 +409,7 @@ describe("Streaming Methods and Responses API", () => { write: true, outfile, format: "esm", - plugins: [esbuildPlugin({ browser: false })], + plugins: [braintrustEsbuildPlugin({ browser: false })], logLevel: "error", absWorkingDir: fixturesDir, preserveSymlinks: true, @@ -461,7 +461,7 @@ describe("Streaming Methods and Responses API", () => { fs.writeFileSync(entryPoint, testCode); - const { esbuildPlugin } = + const { braintrustEsbuildPlugin } = await import("../../src/auto-instrumentations/bundler/esbuild.js"); await esbuild.build({ @@ -470,7 +470,7 @@ describe("Streaming Methods and Responses API", () => { write: true, outfile, format: "esm", - plugins: [esbuildPlugin({ browser: false })], + plugins: [braintrustEsbuildPlugin({ browser: false })], logLevel: "error", absWorkingDir: fixturesDir, preserveSymlinks: true, @@ -554,7 +554,7 @@ describe("Streaming Methods and Responses API", () => { fs.writeFileSync(entryPoint, testCode); - const { esbuildPlugin } = + const { braintrustEsbuildPlugin } = await import("../../src/auto-instrumentations/bundler/esbuild.js"); await esbuild.build({ @@ -563,7 +563,7 @@ describe("Streaming Methods and Responses API", () => { write: true, outfile, format: "esm", - plugins: [esbuildPlugin({ browser: false })], + plugins: [braintrustEsbuildPlugin({ browser: false })], logLevel: "error", absWorkingDir: fixturesDir, preserveSymlinks: true, diff --git a/js/tests/auto-instrumentations/transformation.test.ts b/js/tests/auto-instrumentations/transformation.test.ts index 0a21456cd..18800bdfd 100644 --- a/js/tests/auto-instrumentations/transformation.test.ts +++ b/js/tests/auto-instrumentations/transformation.test.ts @@ -630,7 +630,7 @@ describe("Orchestrion Transformation Tests", () => { it.each([ ["browser", "browser", { browser: true }], - ["legacy-browser", "browser", { useDiagnosticChannelCompatShim: true }], + ["legacy-browser", "browser", { browser: true }], ["edge", "neutral", { browser: true }], ] as const)( "should keep Mastra %s bundles free of Node-only patches", @@ -933,13 +933,7 @@ describe("Orchestrion Transformation Tests", () => { }); it.each([ - [ - "skip", - "legacy browser", - "web", - { useDiagnosticChannelCompatShim: true }, - false, - ], + ["skip", "legacy browser", "web", { browser: true }, false], ["skip", "browser", "web", { browser: true }, false], ["apply", "node", "node", { browser: false }, true], ] as const)( diff --git a/js/tsup.config.ts b/js/tsup.config.ts index 416ef9679..760f06fbf 100644 --- a/js/tsup.config.ts +++ b/js/tsup.config.ts @@ -14,6 +14,7 @@ export default defineConfig([ { entry: { index: "src/node/index.ts", + "instrumentation/index": "src/instrumentation/index.ts", "apply-auto-instrumentation": "src/node/apply-auto-instrumentation-entry.ts", "vitest-evals-reporter": "src/wrappers/vitest-evals/reporter.ts", @@ -32,22 +33,6 @@ export default defineConfig([ define, clean: true, }, - { - entry: ["util/index.ts"], - format: ["cjs", "esm"], - outDir: "util/dist", - external: ["esbuild", "prettier", "typescript", "zod"], - removeNodeProtocol: false, - dts: { - // Split DTS generation to reduce memory usage - compilerOptions: { - skipLibCheck: true, - }, - }, - splitting: true, - define, - clean: true, - }, // Browser/edge entrypoints { entry: { @@ -67,20 +52,6 @@ export default defineConfig([ define, clean: false, }, - { - entry: ["src/instrumentation/index.ts"], - format: ["cjs", "esm"], - outDir: "dist/instrumentation", - external: ["@braintrust/instrumentation-core", "zod"], - dts: { - compilerOptions: { - skipLibCheck: true, - }, - }, - splitting: false, - define, - clean: true, - }, { entry: [ "src/auto-instrumentations/index.ts", diff --git a/js/util/object.ts b/js/util/object.ts index 941c6264a..074aa6229 100644 --- a/js/util/object.ts +++ b/js/util/object.ts @@ -23,7 +23,6 @@ export type OtherExperimentLogFields = { classifications?: Record; metadata: Record; metrics: Record; - datasetRecordId: string; origin: ObjectReferenceType; span_attributes: Record; [ASYNC_SCORING_CONTROL_FIELD]: AsyncScoringControl; @@ -43,7 +42,7 @@ export type ExperimentLogFullArgs = Partial< export type LogFeedbackFullArgs = IdField & Partial< - Omit & { + Omit & { comment: string; source: Source; } diff --git a/js/util/score.ts b/js/util/score.ts index 1c8a4754e..53da7063a 100644 --- a/js/util/score.ts +++ b/js/util/score.ts @@ -40,11 +40,6 @@ export interface Score { name: string; score: number | null; metadata?: Record; - // DEPRECATION_NOTICE: this field is deprecated, as errors are propagated up to the caller. - /** - * @deprecated - */ - error?: unknown; } export type ScorerArgs = { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index c6deed80a..2e0e646d0 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -82,6 +82,9 @@ importers: e2e: devDependencies: + '@braintrust/browser': + specifier: workspace:^ + version: link:../integrations/browser-js '@braintrust/langchain-js': specifier: workspace:^ version: link:../integrations/langchain-js